From d68bfc972f6be757cd05fce46ecc5771f2440bb2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 01:40:18 +0000 Subject: [PATCH 01/37] refact: tsdb --- source/dnode/vnode/CMakeLists.txt | 2 - source/dnode/vnode/src/tsdb/tsdbSma.c | 2203 --------------------- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 128 -- 3 files changed, 2333 deletions(-) delete mode 100644 source/dnode/vnode/src/tsdb/tsdbSma.c delete mode 100644 source/dnode/vnode/src/tsdb/tsdbTDBImpl.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d988f97188..b80adb9a95 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -35,7 +35,6 @@ target_sources( "src/sma/smaTimeRange.c" # tsdb - # "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbFile.c" @@ -45,7 +44,6 @@ target_sources( "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - # "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" "src/tsdb/tsdbSnapshot.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c deleted file mode 100644 index 45b17a0180..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ /dev/null @@ -1,2203 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbSma.h" -#include "tsdb.h" - -static const char *TSDB_SMA_DNAME[] = { - "", // TSDB_SMA_TYPE_BLOCK - "tsma", // TSDB_SMA_TYPE_TIME_RANGE - "rsma", // TSDB_SMA_TYPE_ROLLUP -}; - -#undef _TEST_SMA_PRINT_DEBUG_LOG_ -#define SMA_STORAGE_TSDB_DAYS 30 -#define SMA_STORAGE_TSDB_TIMES 10 -#define SMA_STORAGE_SPLIT_HOURS 24 -#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 -#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds - -#define SMA_STATE_HASH_SLOT 4 -#define SMA_STATE_ITEM_HASH_SLOT 32 - -#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test -#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test - -typedef struct SRSmaInfo SRSmaInfo; -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - -typedef struct SPoolMem { - int64_t size; - struct SPoolMem *prev; - struct SPoolMem *next; -} SPoolMem; - -struct SSmaEnv { - TdThreadRwlock lock; - int8_t type; - TXN txn; - SPoolMem *pPool; - SDiskID did; - TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? - char *path; // relative path - SSmaStat *pStat; -}; - -#define SMA_ENV_LOCK(env) ((env)->lock) -#define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_DID(env) ((env)->did) -#define SMA_ENV_ENV(env) ((env)->dbEnv) -#define SMA_ENV_PATH(env) ((env)->path) -#define SMA_ENV_STAT(env) ((env)->pStat) -#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) - -typedef struct { - STsdb *pTsdb; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int32_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SDBFile dFile; - int32_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int8_t storageLevel; - int8_t days; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef struct { - /** - * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from - * Streaming Module or TSDB local persistence. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. - * - TSDB_SMA_STAT_DROPPED: 1)sma dropped - * N.B. only applicable to tsma - */ - int8_t state; // ETsdbSmaStat - SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma *pSma; // cache schema -} SSmaStatItem; - -#define RSMA_TASK_INFO_HASH_SLOT 8 -struct SRSmaInfo { - void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t -}; - -struct SSmaStat { - union { - SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; - }; - T_REF_DECLARE() -}; -#define SMA_STAT_ITEMS(s) ((s)->smaStatItems) -#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) - -static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { - // Note: free/kill may in RC - qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); - if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { - qDestroyTask(otaskHandle); - } -} - -static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { - if (pInfo->taskInfo[i]) { - tsdbFreeTaskHandle(pInfo->taskInfo[i]); - } - } - return NULL; -} - -// declaration of static functions - -// expired window -static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version); -static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version); -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); -static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); -static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv); -static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); -static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); - -// read data -// TODO: This is the basic params, and should wrap the params to a queryHandle. -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); - -// insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); -static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg); -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); - -static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); -static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, - int8_t level); -// mgmt interface -static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); - -// Pool Memory -static SPoolMem *openPool(); -static void clearPool(SPoolMem *pPool); -static void closePool(SPoolMem *pPool); -static void *poolMalloc(void *arg, size_t size); -static void poolFree(void *arg, void *ptr); - -static int tsdbSmaBeginCommit(SSmaEnv *pEnv); -static int tsdbSmaEndCommit(SSmaEnv *pEnv); - -// implementation -static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { - return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); -} -static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { - return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); -} - -static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockRdlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockWrlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockUnlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); - - pPool->prev = pPool->next = pPool; - pPool->size = 0; - - return pPool; -} - -static void clearPool(SPoolMem *pPool) { - if (!pPool) return; - - SPoolMem *pMem; - - do { - pMem = pPool->next; - - if (pMem == pPool) break; - - pMem->next->prev = pMem->prev; - pMem->prev->next = pMem->next; - pPool->size -= pMem->size; - - taosMemoryFree(pMem); - } while (1); - - assert(pPool->size == 0); -} - -static void closePool(SPoolMem *pPool) { - if (pPool) { - clearPool(pPool); - taosMemoryFree(pPool); - } -} - -static void *poolMalloc(void *arg, size_t size) { - void *ptr = NULL; - SPoolMem *pPool = (SPoolMem *)arg; - SPoolMem *pMem; - - pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); - if (!pMem) { - assert(0); - } - - pMem->size = sizeof(*pMem) + size; - pMem->next = pPool->next; - pMem->prev = pPool; - - pPool->next->prev = pMem; - pPool->next = pMem; - pPool->size += pMem->size; - - ptr = (void *)(&pMem[1]); - return ptr; -} - -static void poolFree(void *arg, void *ptr) { - SPoolMem *pPool = (SPoolMem *)arg; - SPoolMem *pMem; - - pMem = &(((SPoolMem *)ptr)[-1]); - - pMem->next->prev = pMem->prev; - pMem->prev->next = pMem->next; - pPool->size -= pMem->size; - - taosMemoryFree(pMem); -} - -int32_t tsdbInitSma(STsdb *pTsdb) { - // tSma - int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(REPO_META(pTsdb), false)); - if (numOfTSma > 0) { - atomic_store_16(&REPO_TSMA_NUM(pTsdb), (int16_t)numOfTSma); - } - // TODO: rSma - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { - if (pStatItem) { - return atomic_load_8(&pStatItem->state); - } - return TSDB_SMA_STAT_UNKNOWN; -} - -static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { - if (!pStatItem) { - return false; - } - - if (state) { - *state = atomic_load_8(&pStatItem->state); - return *state == TSDB_SMA_STAT_OK; - } - return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; -} - -static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; -} - -static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; -} - -static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); - } -} - -static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); - } -} - -static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); - } -} - -static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); -} - -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did) { - SSmaEnv *pEnv = NULL; - - pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); - if (!pEnv) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMA_ENV_TYPE(pEnv) = smaType; - - int code = taosThreadRwlockInit(&(pEnv->lock), NULL); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(pEnv); - return NULL; - } - - ASSERT(path && (strlen(path) > 0)); - SMA_ENV_PATH(pEnv) = strdup(path); - if (!SMA_ENV_PATH(pEnv)) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - SMA_ENV_DID(pEnv) = did; - - if (tsdbInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - char aname[TSDB_FILENAME_LEN] = {0}; - tfsAbsoluteName(REPO_TFS(pTsdb), did, path, aname); - if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - if (!(pEnv->pPool = openPool())) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - return pEnv; -} - -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) { - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (!(*pEnv)) { - if (!(*pEnv = tsdbNewSmaEnv(pTsdb, smaType, path, did))) { - return TSDB_CODE_FAILED; - } - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Release resources allocated for its member fields, not including itself. - * - * @param pSmaEnv - * @return int32_t - */ -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { - if (pSmaEnv) { - tsdbDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); - taosMemoryFreeClear(pSmaEnv->pStat); - taosMemoryFreeClear(pSmaEnv->path); - taosThreadRwlockDestroy(&(pSmaEnv->lock)); - tsdbCloseDBEnv(pSmaEnv->dbEnv); - closePool(pSmaEnv->pPool); - } -} - -void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { - tsdbDestroySmaEnv(pSmaEnv); - taosMemoryFreeClear(pSmaEnv); - return NULL; -} - -static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_INC(pStat); - tsdbDebug("vgId:%d ref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); - return 0; -} - -static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_DEC(pStat); - tsdbDebug("vgId:%d unref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); - return 0; -} - -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { - ASSERT(pSmaStat != NULL); - - if (*pSmaStat) { // no lock - return TSDB_CODE_SUCCESS; - } - - /** - * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tsdbNew). - * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if - * tsdbInitSmaStat invoked in other multithread environment later. - */ - if (!(*pSmaStat)) { - *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); - if (!(*pSmaStat)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( - RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - - if (!SMA_STAT_INFO_HASH(*pSmaStat)) { - taosMemoryFreeClear(*pSmaStat); - return TSDB_CODE_FAILED; - } - } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - SMA_STAT_ITEMS(*pSmaStat) = - taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - - if (!SMA_STAT_ITEMS(*pSmaStat)) { - taosMemoryFreeClear(*pSmaStat); - return TSDB_CODE_FAILED; - } - } else { - ASSERT(0); - } - } - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (pItem) { - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - } - } - return pItem; -} - -static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { - if (pSmaStatItem) { - tdDestroyTSma(pSmaStatItem->pSma); - taosMemoryFreeClear(pSmaStatItem->pSma); - taosHashCleanup(pSmaStatItem->expiredWindows); - taosMemoryFreeClear(pSmaStatItem); - } - return NULL; -} - -/** - * @brief Release resources allocated for its member fields, not including itself. - * - * @param pSmaStat - * @return int32_t - */ -int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { - if (pSmaStat) { - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL); - while (item) { - SSmaStatItem *pItem = *(SSmaStatItem **)item; - tsdbFreeSmaStatItem(pItem); - item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item); - } - taosHashCleanup(SMA_STAT_ITEMS(pSmaStat)); - } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); - while (infoHash) { - SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; - tsdbFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); - } - taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); - } else { - ASSERT(0); - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { - SSmaEnv *pEnv = NULL; - - // return if already init - switch (smaType) { - case TSDB_SMA_TYPE_TIME_RANGE: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)))) { - return TSDB_CODE_SUCCESS; - } - break; - case TSDB_SMA_TYPE_ROLLUP: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)))) { - return TSDB_CODE_SUCCESS; - } - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - - // init sma env - tsdbLockRepo(pTsdb); - pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)) - : atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); - if (!pEnv) { - char rname[TSDB_FILENAME_LEN] = {0}; - - SDiskID did = {0}; - tfsAllocDisk(REPO_TFS(pTsdb), TFS_PRIMARY_LEVEL, &did); - if (did.level < 0 || did.id < 0) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); - - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rname, did) != TSDB_CODE_SUCCESS) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - - if (tsdbInitSmaEnv(pTsdb, smaType, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - - (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&REPO_TSMA_ENV(pTsdb), pEnv) - : atomic_store_ptr(&REPO_RSMA_ENV(pTsdb), pEnv); - } - tsdbUnlockRepo(pTsdb); - - return TSDB_CODE_SUCCESS; -}; - -static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(REPO_META(pTsdb), indexUid, true); - if (!pSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, - tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pSma = pSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - tsdbWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", REPO_ID(pTsdb), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", REPO_ID(pTsdb), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pTsdb - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) { - tsdbTrace("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb)); - return TSDB_CODE_SUCCESS; - } - - if (!REPO_META(pTsdb)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = REPO_TSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tsdbRefSmaStat(pTsdb, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - pSW = tdFreeTSmaWrapper(pSW); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - tdFreeTSmaWrapper(pSW); - break; - } - if (!pSW || (pTSma->tableUid != pBlock->suid)) { - if (pSW) { - pSW = tdFreeTSmaWrapper(pSW); - } - if (!(pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = REPO_CFG(pTsdb)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); - } else { - tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - REPO_ID(pTsdb), pTSma->indexUid, winSKey); - } - } - } - - tsdbUnRefSmaStat(pTsdb, pStat); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pTsdb - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", REPO_ID(pTsdb), - skey, indexUid); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", REPO_ID(pTsdb), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, REPO_ID(pTsdb), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Judge the tSma storage level - * - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { - // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? - switch (intervalUnit) { - case TIME_UNIT_HOUR: - if (interval < SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MINUTE: - if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_SECOND: - if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MILLISECOND: - if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MICROSECOND: - if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_NANOSECOND: - if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - default: - break; - } - return SMA_STORAGE_LEVEL_TSDB; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = TSDB_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - tsdbCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - STsdb *pTsdb = pSmaH->pTsdb; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pTsdb - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel) { - STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb); - int32_t daysPerFile = pCfg->days; - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; - } - - return daysPerFile; -} - -static int tsdbSmaBeginCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - // start a new txn - tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { - tsdbWarn("tsdbSma tdb begin commit fail"); - return -1; - } - return 0; -} - -static int tsdbSmaEndCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - - // Commit current txn - if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { - tsdbWarn("tsdbSma tdb end commit fail"); - return -1; - } - tdbTxnClose(pTxn); - clearPool(pEnv->pPool); - return 0; -} - -/** - * @brief Insert/Update Time-range-wise SMA data. - * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. - * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. - * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The - * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). - * - The destination file of one data block for some interval is determined by its start TS key. - * - * @param pTsdb - * @param msg - * @return int32_t - */ -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - const SArray *pDataBlocks = (const SArray *)msg; - - // TODO: destroy SSDataBlocks(msg) - - // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus - // the sma data would arrive ahead of the update-expired-window msg. - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = REPO_TSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pSma = pItem->pSma; - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); - int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != TSDB_IVLD_FID) { - tsdbSmaEndCommit(pEnv); - tsdbCloseDBF(&tSmaH.dFile); - } - tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); - if (tsdbOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - tsdbSmaBeginCommit(pEnv); - } - - if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - tsdbWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); - tsdbSmaEndCommit(pEnv); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - REPO_ID(pTsdb), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, pStat, indexUid, skey); - } else { - tsdbWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - REPO_ID(pTsdb), skey, tlen, indexUid); - } - - printf("\n"); - } - } - tsdbSmaEndCommit(pEnv); // TODO: not commit for every insert - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pTsdb - * @param msg - * @return int32_t - */ -static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)); - - // clear local cache - if (pEnv) { - tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tsdbSmaStatIsDropped(pItem)) { - tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tsdbWLockSma(pEnv); - if (tsdbSmaStatIsDropped(pItem)) { - tsdbUnLockSma(pEnv); - tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tsdbSmaStatSetDropped(pItem); - tsdbUnLockSma(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - tsdbDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal); - break; - } - tsdbDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - tsdbDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", REPO_ID(pTsdb), indexUid, nSleep, - refVal); - break; - }; - } - - tsdbFreeSmaStatItem(pItem); - tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) { - STsdb *pTsdb = pSmaH->pTsdb; - - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - const SArray *pDataBlocks = (const SArray *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); - int64_t indexUid = SMA_TEST_INDEX_UID; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pSma = pItem->pSma; - - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); - int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); -#if 0 - int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); - if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { - tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - return TSDB_CODE_FAILED; - } - - if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { - tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - return TSDB_CODE_FAILED; - } - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); -#endif - - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSmaH - * @param pTsdb - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit) { - pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); - pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit); - pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - STsdb *pTsdb = pSmaH->pTsdb; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - int32_t nSmaFs = taosArrayGetSize(smaFs); - - tsdbCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief - * - * @param pTsdb Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tsdbRefSmaStat(pTsdb, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tsdbUnRefSmaStat(pTsdb, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tsdbUnRefSmaStat(pTsdb, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), - querySKey, indexUid); - } else { - tsdbDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey, - indexUid); - } - - STSma *pTSma = pItem->pSma; -#endif - - STSmaReadH tReadH = {0}; - tsdbInitTSmaReadH(&tReadH, pTsdb, pTSma->interval, pTSma->intervalUnit); - tsdbCloseDBF(&tReadH.dFile); - - tsdbUnRefSmaStat(pTsdb, pStat); - - tsdbInitTSmaFile(&tReadH, indexUid, querySKey); - if (tsdbOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 1; - tsdbEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), - tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - tsdbCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tsdbSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tsdbGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - tsdbCloseDBF(&tReadH.dFile); - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { - SSmaCfg vCreateSmaReq = {0}; - if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); - return -1; - } - - tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); - - // record current timezone of server side - vCreateSmaReq.tSma.timezoneInt = tsTimezone; - - if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { - // TODO: handle error - tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); - tdDestroyTSma(&vCreateSmaReq.tSma); - return -1; - } - - tsdbTSmaAdd(pTsdb, 1); - - tdDestroyTSma(&vCreateSmaReq.tSma); - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tdDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // - - if (metaDropTSma(REPO_META(pTsdb), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tsdbDropTSmaData(pTsdb, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tsdbTSmaSub(pTsdb, 1); - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. - * - * @param pTsdb - * @param pMeta - * @param pReq - * @return int32_t - */ -int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) { - if (!pReq->rollup) { - tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - SRSmaParam *param = &pReq->pRSmaParam; - - if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { - tsdbWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); - if (pRSmaInfo) { - tsdbWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); - if (!pRSmaInfo) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); - if (!pReadHandle) { - taosMemoryFree(pRSmaInfo); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - SReadHandle handle = { - .reader = pReadHandle, - .meta = pMeta, - .pMsgCb = pMsgCb, - }; - - if (param->qmsg1) { - pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle); - if (!pRSmaInfo->taskInfo[0]) { - taosMemoryFree(pRSmaInfo); - taosMemoryFree(pReadHandle); - return TSDB_CODE_FAILED; - } - } - - if (param->qmsg2) { - pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle); - if (!pRSmaInfo->taskInfo[1]) { - taosMemoryFree(pRSmaInfo); - taosMemoryFree(pReadHandle); - return TSDB_CODE_FAILED; - } - } - - if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->suid); - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief store suid/[uids], prefer to use array and then hash - * - * @param pStore - * @param suid - * @param uid - * @return int32_t - */ -static int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { - // prefer to store suid/uids in array - if ((suid == pStore->suid) || (pStore->suid == 0)) { - if (pStore->suid == 0) { - pStore->suid = suid; - } - if (uid) { - if (!pStore->tbUids) { - if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - if (!taosArrayPush(pStore->tbUids, uid)) { - return TSDB_CODE_FAILED; - } - } - } else { - // store other suid/uids in hash when multiple stable/table included in 1 batch of request - if (!pStore->uidHash) { - pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if (!pStore->uidHash) { - return TSDB_CODE_FAILED; - } - } - if (uid) { - SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); - if (uidArray && ((uidArray = *(SArray **)uidArray))) { - taosArrayPush(uidArray, uid); - } else { - SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); - if (!pUidArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - if (!taosArrayPush(pUidArray, uid)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { - return TSDB_CODE_FAILED; - } - } - } else { - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { - return TSDB_CODE_FAILED; - } - } - } - return TSDB_CODE_SUCCESS; -} - -void tsdbUidStoreDestory(STbUidStore *pStore) { - if (pStore) { - if (pStore->uidHash) { - if (pStore->tbUids) { - // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys. - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { - SArray *arr = *(SArray **)pIter; - taosArrayDestroy(arr); - pIter = taosHashIterate(pStore->uidHash, pIter); - } - } - taosHashCleanup(pStore->uidHash); - } - taosArrayDestroy(pStore->tbUids); - } -} - -void *tsdbUidStoreFree(STbUidStore *pStore) { - if (pStore) { - tsdbUidStoreDestory(pStore); - taosMemoryFree(pStore); - } - return NULL; -} - -/** - * @brief fetch suid/uids when create child tables of rollup SMA - * - * @param pTsdb - * @param ppStore - * @param suid - * @param uid - * @return int32_t - */ -int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) { - SSmaEnv *pEnv = REPO_RSMA_ENV((STsdb *)pTsdb); - - // only applicable to rollup SMA ctables - if (!pEnv) { - return TSDB_CODE_SUCCESS; - } - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - return TSDB_CODE_FAILED; - } - - // info cached when create rsma stable and return directly for non-rsma ctables - if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) { - return TSDB_CODE_SUCCESS; - } - - ASSERT(ppStore != NULL); - - if (!(*ppStore)) { - if (tsdbUidStoreInit(ppStore) != 0) { - return TSDB_CODE_FAILED; - } - } - - if (tsdbUidStorePut(*ppStore, suid, &uid) != 0) { - *ppStore = tsdbUidStoreFree(*ppStore); - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - if (!suid || !tbUids) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64, REPO_ID(pTsdb), *suid); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - return TSDB_CODE_FAILED; - } - - if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { - tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), - pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - - if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { - tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), - pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { - if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { - return TSDB_CODE_SUCCESS; - } - - if (tsdbUpdateTbUidListImpl(pTsdb, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { - tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - SArray *pTbUids = *(SArray **)pIter; - - if (tsdbUpdateTbUidListImpl(pTsdb, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { - taosHashCancelIterate(pStore->uidHash, pIter); - return TSDB_CODE_FAILED; - } - - pIter = taosHashIterate(pStore->uidHash, pIter); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - - terrno = TSDB_CODE_SUCCESS; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - - if (!pBlock) break; - tsdbUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, - int8_t level) { - SArray *pResult = NULL; - - if (!taskInfo) { - tsdbDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, REPO_ID(pTsdb), level, suid); - return TSDB_CODE_SUCCESS; - } - - tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, - suid); - - qSetStreamInput(taskInfo, pMsg, inputType, false); - while (1) { - SSDataBlock *output = NULL; - uint64_t ts; - if (qExecTask(taskInfo, &output, &ts) < 0) { - ASSERT(false); - } - if (!output) { - break; - } - if (!pResult) { - pResult = taosArrayInit(0, sizeof(SSDataBlock)); - if (!pResult) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - - taosArrayPush(pResult, output); - } - - if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); - STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); - SSubmitReq *pReq = NULL; - if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), suid) != 0) { - taosArrayDestroy(pResult); - return TSDB_CODE_FAILED; - } - if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { - taosArrayDestroy(pResult); - taosMemoryFreeClear(pReq); - return TSDB_CODE_FAILED; - } - taosMemoryFreeClear(pReq); - } else { - tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); - } - - taosArrayDestroy(pResult); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - if (!pEnv) { - // only applicable when rsma env exists - return TSDB_CODE_SUCCESS; - } - - ASSERT(uid != 0); // TODO: remove later - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), suid); - return TSDB_CODE_SUCCESS; - } - if (!pRSmaInfo->taskInfo[0]) { - tsdbDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, REPO_ID(pTsdb), suid); - return TSDB_CODE_SUCCESS; - } - - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - // TODO: use the proper schema instead of 1, and cache STSchema in cache - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1); - if (!pTSchema) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return TSDB_CODE_FAILED; - } - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); - taosMemoryFree(pTSchema); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - if (!pEnv) { - // only applicable when rsma env exists - return TSDB_CODE_SUCCESS; - } - - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - STbUidStore uidStore = {0}; - tsdbFetchSubmitReqSuids(pMsg, &uidStore); - - if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); - - void *pIter = taosHashIterate(uidStore.uidHash, NULL); - while (pIter) { - tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); - pIter = taosHashIterate(uidStore.uidHash, pIter); - } - - tsdbUidStoreDestory(&uidStore); - } - } - return TSDB_CODE_SUCCESS; -} - -#if 0 -/** - * @brief Get the start TS key of the last data block of one interval/sliding. - * - * @param pTsdb - * @param param - * @param result - * @return int32_t - * 1) Return 0 and fill the result if the check procedure is normal; - * 2) Return -1 if error occurs during the check procedure. - */ -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, void *smaIndex, void *result) { - const char *procedure = ""; - if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { - return -1; - } - // fill the result - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Remove the tSma data files related to param between pWin. - * - * @param pTsdb - * @param param - * @param pWin - * @return int32_t - */ -int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { - // for ("tSmaFiles of param-interval-sliding between pWin") { - // // remove the tSmaFile - // } - return TSDB_CODE_SUCCESS; -} -#endif - -// TODO: Who is responsible for resource allocate and release? -int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertTSmaDataImpl(pTsdb, indexUid, msg)) < 0) { - tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, pMsg, version)) < 0) { - tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, querySKey, nMaxResult)) < 0) { - tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { - tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c deleted file mode 100644 index a553f32bee..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define ALLOW_FORBID_FUNC - -#include "tsdb.h" - -int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) { - int ret = 0; - - if (path == NULL) return -1; - - ret = tdbOpen(path, 4096, 256, ppEnv); // use as param - - if (ret != 0) { - tsdbError("Failed to create tsdb db env, ret = %d", ret); - return -1; - } - - return 0; -} - -int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } - -static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { - const SSmaKey *pKey1 = (const SSmaKey *)arg1; - const SSmaKey *pKey2 = (const SSmaKey *)arg2; - - ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); - - if (pKey1->skey < pKey2->skey) { - return -1; - } else if (pKey1->skey > pKey2->skey) { - return 1; - } - if (pKey1->groupId < pKey2->groupId) { - return -1; - } else if (pKey1->groupId > pKey2->groupId) { - return 1; - } - - return 0; -} - -static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { - int ret; - tdb_cmpr_fn_t compFunc; - - // Create a database - compFunc = tsdbSmaKeyCmpr; - ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); - - return 0; -} - -static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } - -int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) { - // TEnv is shared by a group of SDBFile - if (!pEnv || !pDBF) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - // Open DBF - if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - tsdbCloseDBDb(pDBF->pDB); - return -1; - } - - return 0; -} - -int32_t tsdbCloseDBF(SDBFile *pDBF) { - int32_t ret = 0; - if (pDBF->pDB) { - ret = tsdbCloseDBDb(pDBF->pDB); - pDBF->pDB = NULL; - } - taosMemoryFreeClear(pDBF->path); - return ret; -} - -int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { - int32_t ret; - - ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); - if (ret < 0) { - tsdbError("Failed to create insert sma data into db, ret = %d", ret); - return -1; - } - - return 0; -} - -void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { - void *pVal = NULL; - int ret; - - ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); - - if (ret < 0) { - tsdbError("Failed to get sma data from db, ret = %d", ret); - return NULL; - } - - ASSERT(*valLen >= 0); - - // TODO: lock? - // TODO: Would the key/value be destoryed during return the data? - // TODO: How about the key is updated while value length is changed? The original value buffer would be freed - // automatically? - - return pVal; -} \ No newline at end of file From 0c31c89b38f52a61476ef51b98bad5ce59f03827 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 02:48:45 +0000 Subject: [PATCH 02/37] refact more --- include/common/tdataformat.h | 21 +++++++ source/common/src/tdataformat.c | 103 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 957b93d359..986ef5edb9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSchema SSchema; typedef struct STColumn STColumn; typedef struct STSchema STSchema; +typedef struct SValue SValue; typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; @@ -116,6 +117,26 @@ struct STSRowBuilder { STSRow2 row; }; +struct SValue { + union { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + TSKEY ts; + float f; + double d; + struct { + uint32_t nData; + uint8_t *pData; + }; + }; +}; + typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; struct SColVal { EColValT type; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34e78c21db..f762c48f44 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,6 +43,109 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); +// SValue +static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { + int32_t n = 0; + + if (IS_VAR_DATA_TYPE(type)) { + n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); + } else { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0); + break; + case TSDB_DATA_TYPE_TINYINT: + n += tPutI8(p ? p + n : p, pValue->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tPutI16(p ? p + n : p, pValue->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tPutI32(p ? p + n : p, pValue->i32); + break; + case TSDB_DATA_TYPE_BIGINT: + n += tPutI64(p ? p + n : p, pValue->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tPutFloat(p ? p + n : p, pValue->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tPutDouble(p ? p + n : p, pValue->d); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + n += tPutI64(p ? p + n : p, pValue->ts); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tPutU8(p ? p + n : p, pValue->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tPutU16(p ? p + n : p, pValue->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tPutU32(p ? p + n : p, pValue->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tPutU64(p ? p + n : p, pValue->u64); + break; + default: + ASSERT(0); + } + } + + return n; +} + +static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { + int32_t n = 0; + + if (IS_VAR_DATA_TYPE(type)) { + n += tGetBinary(p, pValue ? &pValue->pData : NULL, pValue ? &pValue->nData : NULL); + } else { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + n += tGetI8(p, &pValue->i8); + break; + case TSDB_DATA_TYPE_TINYINT: + n += tGetI8(p, &pValue->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tGetI16(p, &pValue->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tGetI32(p, &pValue->i32); + break; + case TSDB_DATA_TYPE_BIGINT: + n += tGetI64(p, &pValue->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tGetFloat(p, &pValue->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tGetDouble(p, &pValue->d); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + n += tGetI64(p, &pValue->ts); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tGetU8(p, &pValue->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tGetU16(p, &pValue->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tGetU32(p, &pValue->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tGetU64(p, &pValue->u64); + break; + default: + ASSERT(0); + } + } + + return n; +} + // STSRow2 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; From 010f1d2e2f7a080feb995aac67e8a5c54b829c3a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 02:59:55 +0000 Subject: [PATCH 03/37] more refact --- source/common/src/tdataformat.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f762c48f44..c57de4e2fc 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -103,40 +103,40 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { } else { switch (type) { case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, &pValue->i8); + n += tGetI8(p, pValue ? &pValue->i8 : NULL); break; case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, &pValue->i8); + n += tGetI8(p, pValue ? &pValue->i8 : NULL); break; case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pValue->i16); + n += tGetI16(p, pValue ? &pValue->i16 : NULL); break; case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pValue->i32); + n += tGetI32(p, pValue ? &pValue->i32 : NULL); break; case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pValue->i64); + n += tGetI64(p, pValue ? &pValue->i64 : NULL); break; case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pValue->f); + n += tGetFloat(p, pValue ? &pValue->f : NULL); break; case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pValue->d); + n += tGetDouble(p, pValue ? &pValue->d : NULL); break; case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, &pValue->ts); + n += tGetI64(p, pValue ? &pValue->ts : NULL); break; case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pValue->u8); + n += tGetU8(p, pValue ? &pValue->u8 : NULL); break; case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pValue->u16); + n += tGetU16(p, pValue ? &pValue->u16 : NULL); break; case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pValue->u32); + n += tGetU32(p, pValue ? &pValue->u32 : NULL); break; case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pValue->u64); + n += tGetU64(p, pValue ? &pValue->u64 : NULL); break; default: ASSERT(0); From d6f02761a02e0a8c430919500104c313c8bbefec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:08:29 +0000 Subject: [PATCH 04/37] more refact --- include/common/tdataformat.h | 19 ++++++++++++------- source/common/src/tdataformat.c | 12 +++++++++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 986ef5edb9..ed3a911c13 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -42,11 +42,13 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SColVal -#define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) -#define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) -#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) +// #define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) +// #define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) +// #define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) // STSRow2 +int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); @@ -54,11 +56,13 @@ void tTSRowFree(STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder +#if 0 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema); void tTSRowBuilderClear(STSRowBuilder *pBuilder); void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); +#endif // STagVal static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, @@ -137,11 +141,12 @@ struct SValue { }; }; -typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; struct SColVal { - EColValT type; - uint32_t nData; - uint8_t *pData; + int8_t isNone; + int8_t isNull; + int8_t type; + int16_t cid; + SValue value; }; struct STagVal { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c57de4e2fc..653381925c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -146,7 +146,13 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -// STSRow2 +// STSRow2 ======================================================================== +int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { + int32_t code = 0; + // TODO + return code; +} + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; @@ -215,6 +221,7 @@ void tTSRowFree(STSRow2 *pRow) { } int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { +#if 0 uint32_t n; uint8_t *p; uint8_t v; @@ -310,6 +317,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal } } +#endif return 0; } @@ -354,6 +362,7 @@ void tTSchemaDestroy(STSchema *pTSchema) { } // STSRowBuilder +#if 0 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; @@ -612,6 +621,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { return 0; } +#endif static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { From 2baee09b5863d399974c90d9423238479aac8c6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:34:38 +0000 Subject: [PATCH 05/37] more --- include/common/tdataformat.h | 1 - source/common/src/tdataformat.c | 47 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ed3a911c13..c75c5b41f2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -144,7 +144,6 @@ struct SValue { struct SColVal { int8_t isNone; int8_t isNull; - int8_t type; int16_t cid; SValue value; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 653381925c..cea633a8d9 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -44,7 +44,7 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); // SValue -static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { +static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -95,7 +95,7 @@ static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { +static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -146,6 +146,49 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } +// SColVal +static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { + int32_t n = 0; + + ASSERT(pColVal->isNone == 0); + if (isTuple) { + ASSERT(pColVal->isNull == 0); + n += tPutValue(p ? p + n : p, &pColVal->value, type); + } else { + if (pColVal->isNull) { + // -cid means NULL + n += tPutI16v(p ? p + n : p, -pColVal->cid); + } else { + n += tPutI16v(p ? p + n : p, pColVal->cid); + n += tPutValue(p ? p + n : p, &pColVal->value, type); + } + } + + return n; +} + +static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { + int32_t n = 0; + int16_t cid; + + if (isTuple) { + n += tGetValue(p + n, pColVal ? &pColVal->value : NULL, type); + } else { + n += tGetI16v(p + n, &cid); + if (cid < 0) { + if (pColVal) { + pColVal->isNull = 1; + pColVal->cid = -cid; + } + } else { + if (pColVal) pColVal->cid = cid; + n += tGetValue(p ? p + n : p, pColVal ? &pColVal->value : NULL, type); + } + } + + return n; +} + // STSRow2 ======================================================================== int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; From 607ae9e6a4c00271b675b5cc841520956c364bf3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:44:44 +0000 Subject: [PATCH 06/37] refact more --- source/common/src/tdataformat.c | 43 +++++++++++++++------------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cea633a8d9..2994411693 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -99,44 +99,44 @@ static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { - n += tGetBinary(p, pValue ? &pValue->pData : NULL, pValue ? &pValue->nData : NULL); + n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); } else { switch (type) { case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, pValue ? &pValue->i8 : NULL); + n += tGetI8(p, &pValue->i8); break; case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, pValue ? &pValue->i8 : NULL); + n += tGetI8(p, &pValue->i8); break; case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, pValue ? &pValue->i16 : NULL); + n += tGetI16(p, &pValue->i16); break; case TSDB_DATA_TYPE_INT: - n += tGetI32(p, pValue ? &pValue->i32 : NULL); + n += tGetI32(p, &pValue->i32); break; case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, pValue ? &pValue->i64 : NULL); + n += tGetI64(p, &pValue->i64); break; case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, pValue ? &pValue->f : NULL); + n += tGetFloat(p, &pValue->f); break; case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, pValue ? &pValue->d : NULL); + n += tGetDouble(p, &pValue->d); break; case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, pValue ? &pValue->ts : NULL); + n += tGetI64(p, &pValue->ts); break; case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, pValue ? &pValue->u8 : NULL); + n += tGetU8(p, &pValue->u8); break; case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, pValue ? &pValue->u16 : NULL); + n += tGetU16(p, &pValue->u16); break; case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, pValue ? &pValue->u32 : NULL); + n += tGetU32(p, &pValue->u32); break; case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, pValue ? &pValue->u64 : NULL); + n += tGetU64(p, &pValue->u64); break; default: ASSERT(0); @@ -169,20 +169,17 @@ static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { int32_t n = 0; - int16_t cid; + memset(pColVal, 0, sizeof(*pColVal)); if (isTuple) { - n += tGetValue(p + n, pColVal ? &pColVal->value : NULL, type); + n += tGetValue(p + n, &pColVal->value, type); } else { - n += tGetI16v(p + n, &cid); - if (cid < 0) { - if (pColVal) { - pColVal->isNull = 1; - pColVal->cid = -cid; - } + n += tGetI16v(p + n, &pColVal->cid); + if (pColVal->cid < 0) { + pColVal->isNull = 1; + pColVal->cid = -pColVal->cid; } else { - if (pColVal) pColVal->cid = cid; - n += tGetValue(p ? p + n : p, pColVal ? &pColVal->value : NULL, type); + n += tGetValue(p + n, &pColVal->value, type); } } From 0e401a346f24420da72d961f7c20f59fe4a2b69e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 05:37:10 +0000 Subject: [PATCH 07/37] more --- include/common/tmsg.h | 16 ++++++++++++++++ include/common/tmsgdef.h | 2 ++ source/dnode/vnode/src/inc/tsdb.h | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 37e4cc5e8c..54676a0382 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2632,6 +2632,22 @@ typedef struct { int32_t tEncodeSVSubmitReq(SEncoder* pCoder, const SVSubmitReq* pReq); int32_t tDecodeSVSubmitReq(SDecoder* pCoder, SVSubmitReq* pReq); +// TDMT_VND_DELETE +typedef struct { + TSKEY sKey; + TSKEY eKey; + + // super table + char* stbName; + + // child/normal + char* tbName; +} SVDeleteReq; + +typedef struct { + // TODO +} SVDeleteRsp; + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..62517a9a14 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -226,6 +226,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "vnode-delete-data", SVDeleteReq, SVDeleteRsp) + // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2e4ff6a4ab..e5f163a03b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -33,6 +33,7 @@ extern "C" { // clang-format on // tsdbMemTable ================ +typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; typedef struct STsdbMemTable STsdbMemTable; typedef struct SMergeInfo SMergeInfo; @@ -845,6 +846,11 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } +struct STsdbRow { + int64_t version; + STSRow row; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 9b9a431b50..99a4d78c6f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -321,7 +321,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } - if(pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name); + if (pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name); if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schemaRow.version; From d68c15c5fb4b7d07ae3e21c4496bed894efa0ffd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 05:44:33 +0000 Subject: [PATCH 08/37] make it compile --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 025b2ab580..cb9bf00ed7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,6 +15,7 @@ #include "tsdb.h" +#if 0 typedef struct SMemTable SMemTable; typedef struct SMemData SMemData; typedef struct SMemSkipList SMemSkipList; @@ -377,4 +378,5 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} \ No newline at end of file +} +#endif \ No newline at end of file From e86db5fa79786aa794c679ef8ec1e97dbd31d78a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 06:57:34 +0000 Subject: [PATCH 09/37] more --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/tsdb/tsdbDelete.c | 14 +++++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 63 --------------------- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 6 +- 4 files changed, 17 insertions(+), 68 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbDelete.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b80adb9a95..dd51e2c911 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -41,7 +41,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbMemTable2.c" + # "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbDelete.c b/source/dnode/vnode/src/tsdb/tsdbDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 99a4d78c6f..7b3882aa75 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -142,69 +142,6 @@ int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter } else { fKey = tdGetKey(filterKeys[filterIter]); } -#if 0 - } else if (fKey > rowKey) { - if (isRowDel) { - pMergeInfo->rowsDeleteFailed++; - } else { - if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - - pMergeInfo->rowsInserted++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = TD_ROW_IS_DELETED(row); - } - } else { - if (isRowDel) { - ASSERT(!keepDup); - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsDeleteSucceed++; - pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - if (keepDup) { - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); - } - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = TD_ROW_IS_DELETED(row); - } - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } -#endif #if 1 } else if (fKey > rowKey) { if (isRowDel) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index cb9bf00ed7..690e1baf73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,7 +15,6 @@ #include "tsdb.h" -#if 0 typedef struct SMemTable SMemTable; typedef struct SMemData SMemData; typedef struct SMemSkipList SMemSkipList; @@ -100,7 +99,7 @@ static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); -// SMemTable +// SMemTable ======================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -378,5 +377,4 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} -#endif \ No newline at end of file +} \ No newline at end of file From 3eefc938302af19e06fe24cead3ae6ec77967120 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 07:34:39 +0000 Subject: [PATCH 10/37] more code --- source/dnode/vnode/src/inc/tsdb.h | 10 ++++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 65 ++++++++++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e5f163a03b..2283f919bc 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,11 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +typedef struct TSDBKEY TSDBKEY; + +// tsdbMemTable2.c ============================================================================================== +typedef struct SMemTable SMemTable; + // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; @@ -851,6 +856,11 @@ struct STsdbRow { STSRow row; }; +struct TSDBKEY { + int64_t version; + TSKEY ts; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 690e1baf73..f8304be783 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,27 +15,45 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; -typedef struct SMemSkipListCurosr SMemSkipListCurosr; +typedef struct SMemData SMemData; +// typedef struct SMemSkipList SMemSkipList; +// typedef struct SMemSkipListNode SMemSkipListNode; +// typedef struct SMemSkipListCurosr SMemSkipListCurosr; -#define SL_MAX_LEVEL 5 +struct SMemData { + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SMemData *pHashNext; +}; struct SMemTable { - STsdb *pTsdb; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - int32_t nHash; - int32_t nBucket; - SMemData **pBuckets; - SMemSkipListCurosr *pSlc; + STsdb *pTsdb; + int32_t nRef; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SArray *pArray; }; +int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbMemTableDestroy(SMemTable *pMemTable) { + int32_t code = 0; + // TODO + return code; +} + +#if 0 + +#define SL_MAX_LEVEL 5 + struct SMemSkipListNode { int8_t level; SMemSkipListNode *forwards[1]; // Windows does not allow 0 @@ -49,18 +67,6 @@ struct SMemSkipList { SMemSkipListNode pHead[1]; // Windows does not allow 0 }; -struct SMemData { - SMemData *pHashNext; - tb_uid_t suid; - tb_uid_t uid; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - SMemSkipList sl; -}; - struct SMemSkipListCurosr { SMemSkipList *pSl; SMemSkipListNode *pNodes[SL_MAX_LEVEL]; @@ -377,4 +383,5 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} \ No newline at end of file +} +#endif \ No newline at end of file From 31224d1263c666980fcf055891d6f4680f4f4256 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 07:36:21 +0000 Subject: [PATCH 11/37] more --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2283f919bc..19c05bd9cb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -45,7 +45,7 @@ typedef struct SMergeInfo SMergeInfo; typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); -void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); +void tsdbMemTableDestroy(STsdbMemTable *pMemTable); int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 88d8ee9f92..0a85cb4638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -238,7 +238,7 @@ static void tsdbStartCommit(STsdb *pRepo) { static void tsdbEndCommit(STsdb *pTsdb, int eno) { tsdbEndFSTxn(pTsdb); - tsdbMemTableDestroy(pTsdb, pTsdb->imem); + tsdbMemTableDestroy(pTsdb->imem); pTsdb->imem = NULL; tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 7b3882aa75..350e723541 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -60,7 +60,7 @@ int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { return 0; } -void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) { +void tsdbMemTableDestroy(STsdbMemTable *pMemTable) { if (pMemTable) { taosHashCleanup(pMemTable->pHashIdx); SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx); From e0b4319bd3596820969d058e924ead941e0bfd96 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 08:00:49 +0000 Subject: [PATCH 12/37] more --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/inc/tsdb.h | 11 +++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 90 ++++++++------------- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dd51e2c911..b80adb9a95 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -41,7 +41,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" - # "src/tsdb/tsdbMemTable2.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 19c05bd9cb..81dcea9917 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -33,10 +33,14 @@ extern "C" { // clang-format on typedef struct TSDBKEY TSDBKEY; +typedef struct SDelOp SDelOp; // tsdbMemTable2.c ============================================================================================== typedef struct SMemTable SMemTable; +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable); +void tsdbMemTableDestroy2(SMemTable *pMemTable); + // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; @@ -861,6 +865,13 @@ struct TSDBKEY { TSKEY ts; }; +struct SDelOp { + int64_t version; + TSKEY sKey; // included + TSKEY eKey; // included + SDelOp *pNext; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index f8304be783..ef130d1af6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -21,12 +21,12 @@ typedef struct SMemData SMemData; // typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SDelOp *pDelList; }; struct SMemTable { @@ -38,16 +38,39 @@ struct SMemTable { SArray *pArray; }; -int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { - int32_t code = 0; - // TODO +// SMemTable ============================================== +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { + int32_t code = 0; + SMemTable *pMemTable = NULL; + + pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); + if (pMemTable == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pMemTable->pTsdb = pTsdb; + pMemTable->nRef = 1; + pMemTable->minKey = (TSDBKEY){.version = -1, .ts = TSKEY_MAX}; + pMemTable->maxKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MIN}; + pMemTable->nRows = 0; + pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); + if (pMemTable->pArray == NULL) { + taosMemoryFree(pMemTable); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + *ppMemTable = pMemTable; + return code; + +_err: + *ppMemTable = NULL; return code; } -int32_t tsdbMemTableDestroy(SMemTable *pMemTable) { - int32_t code = 0; +void tsdbMemTableDestroy2(SMemTable *pMemTable) { + ASSERT(0); // TODO - return code; } #if 0 @@ -106,49 +129,6 @@ static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); // SMemTable ======================== -int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { - SMemTable *pMemTb = NULL; - - pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); - if (pMemTb == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMemTb->pTsdb = pTsdb; - pMemTb->minKey = TSKEY_MAX; - pMemTb->maxKey = TSKEY_MIN; - pMemTb->minVer = -1; - pMemTb->maxVer = -1; - pMemTb->nRows = 0; - pMemTb->nHash = 0; - pMemTb->nBucket = 1024; - pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); - if (pMemTb->pBuckets == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pMemTb); - return -1; - } - if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pMemTb->pBuckets); - taosMemoryFree(pMemTb); - } - - *ppMemTb = pMemTb; - return 0; -} - -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { - if (pMemTb) { - // loop to destroy the contents (todo) - tsdbMemSkipListCursorDestroy(pMemTb->pSlc); - taosMemoryFree(pMemTb->pBuckets); - taosMemoryFree(pMemTb); - } - return 0; -} - int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { SMemData *pMemData; STsdb *pTsdb = pMemTb->pTsdb; From 81ef708c2ada4cc9a34193884c1ed0f3b8e77f94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 08:42:46 +0000 Subject: [PATCH 13/37] more --- source/dnode/vnode/src/inc/tsdb.h | 8 +-- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 62 +++++++++++++++++++-- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 81dcea9917..de69096902 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,7 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; typedef struct SDelOp SDelOp; @@ -855,9 +856,9 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } -struct STsdbRow { - int64_t version; - STSRow row; +struct TSDBROW { + int64_t version; + STSRow2 *pRow; }; struct TSDBKEY { @@ -869,7 +870,6 @@ struct SDelOp { int64_t version; TSKEY sKey; // included TSKEY eKey; // included - SDelOp *pNext; }; #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ef130d1af6..4494dafc44 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -26,7 +26,7 @@ struct SMemData { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SDelOp *pDelList; + SArray *aDelOp; // SArray }; struct SMemTable { @@ -35,9 +35,11 @@ struct SMemTable { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SArray *pArray; + SArray *pArray; // SArray }; +static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); + // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -69,11 +71,61 @@ _err: } void tsdbMemTableDestroy2(SMemTable *pMemTable) { - ASSERT(0); - // TODO + taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/); + taosMemoryFree(pMemTable); } -#if 0 +int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) { + int32_t code = 0; + SMemData *pMemData; + + // check if table exists + { + // TODO + } + + code = tsdbGetOrCreateTbData(pTsdb, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); + if (code) { + tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _err; + } + + // do insert + + return code; + +_err: + return code; +} + +int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { + int32_t code = 0; + SMemData *pMemData = NULL; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + + // search + pMemData = (SMemData *)taosbsearch(NULL, pMemTable->pArray->pData, taosArrayGetSize(pMemTable->pArray), + sizeof(SMemData *), NULL, TD_GE); + + if (pMemData == NULL) { + // not found, create one + } + + *ppMemData = pMemData; + return code; + +_err: + *ppMemData = NULL; + return code; +} + +#if 0 //==================================================================================== #define SL_MAX_LEVEL 5 From 670a90e55a656c2c4c49bcec690a21ce4828fafc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 10:11:32 +0000 Subject: [PATCH 14/37] feat: tsdb multi-version --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 127 +++++++++++++++++--- 2 files changed, 109 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index de69096902..fe6ecdcbb1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -870,6 +870,7 @@ struct SDelOp { int64_t version; TSKEY sKey; // included TSKEY eKey; // included + SDelOp *pNext; }; #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 4494dafc44..049364e9d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -26,7 +26,8 @@ struct SMemData { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SArray *aDelOp; // SArray + SDelOp *delOpHead; + SDelOp *delOpTail; }; struct SMemTable { @@ -38,7 +39,8 @@ struct SMemTable { SArray *pArray; // SArray }; -static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); +static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); +static int memDataPCmprFn(const void *p1, const void *p2); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -52,8 +54,8 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { } pMemTable->pTsdb = pTsdb; pMemTable->nRef = 1; - pMemTable->minKey = (TSDBKEY){.version = -1, .ts = TSKEY_MAX}; - pMemTable->maxKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MIN}; + pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; + pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; pMemTable->nRows = 0; pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); if (pMemTable->pArray == NULL) { @@ -76,15 +78,17 @@ void tsdbMemTableDestroy2(SMemTable *pMemTable) { } int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) { - int32_t code = 0; - SMemData *pMemData; + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO + SMemData *pMemData; + + ASSERT(pMemTable); - // check if table exists { - // TODO + // check if table exists (todo) } - code = tsdbGetOrCreateTbData(pTsdb, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); + code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); if (code) { tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _err; @@ -99,24 +103,90 @@ _err: } int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { - int32_t code = 0; - // TODO + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO + SMemData *pMemData; + SVBufPool *pPool = pTsdb->pVnode->inUse; + + ASSERT(pMemTable); + + { + // check if table exists (todo) + } + + code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData); + if (code) { + goto _err; + } + + // do delete + SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp)); + if (pDelOp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pDelOp->version = version; + pDelOp->sKey = sKey; + pDelOp->eKey = eKey; + pDelOp->pNext = NULL; + if (pMemData->delOpHead == NULL) { + ASSERT(pMemData->delOpTail == NULL); + pMemData->delOpHead = pMemData->delOpTail = pDelOp; + } else { + pMemData->delOpTail->pNext = pDelOp; + pMemData->delOpTail = pDelOp; + } + + { + // update the state of pMemTable, pMemData, last and lastrow (todo) + } + + tsdbDebug("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + " since %s", + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); + return code; + +_err: + tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + " since %s", + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; } -static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { +static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { int32_t code = 0; + int32_t idx = 0; + SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid}; SMemData *pMemData = NULL; - SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; - // search - pMemData = (SMemData *)taosbsearch(NULL, pMemTable->pArray->pData, taosArrayGetSize(pMemTable->pArray), - sizeof(SMemData *), NULL, TD_GE); - - if (pMemData == NULL) { - // not found, create one + // get + idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); + if (idx >= 0) { + pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx); + if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit; } + // create + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + if (pMemData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pMemData->suid = suid; + pMemData->uid = uid; + pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; + pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; + pMemData->nRows = 0; + pMemData->delOpHead = pMemData->delOpTail = NULL; + + if (idx < 0) idx = 0; + if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + +_exit: *ppMemData = pMemData; return code; @@ -125,6 +195,25 @@ _err: return code; } +static int memDataPCmprFn(const void *p1, const void *p2) { + SMemData *pMemData1 = *(SMemData **)p1; + SMemData *pMemData2 = *(SMemData **)p2; + + if (pMemData1->suid < pMemData2->suid) { + return -1; + } else if (pMemData1->suid > pMemData2->suid) { + return 1; + } + + if (pMemData1->uid < pMemData2->uid) { + return -1; + } else if (pMemData1->uid > pMemData2->uid) { + return 1; + } + + return 0; +} + #if 0 //==================================================================================== #define SL_MAX_LEVEL 5 From 671f613c02ad5df7bb5ad145f3c1db17a3a58071 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 10:20:58 +0000 Subject: [PATCH 15/37] feat: tsdb multi-version --- source/dnode/vnode/src/inc/tsdb.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index fe6ecdcbb1..4f22ce768f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -857,8 +857,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { } struct TSDBROW { - int64_t version; - STSRow2 *pRow; + int64_t version; + STSRow2 tsRow; }; struct TSDBKEY { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 049364e9d7..aab190b0bb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -81,6 +81,7 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit int32_t code = 0; SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO SMemData *pMemData; + TSDBROW row = {.version = version}; ASSERT(pMemTable); @@ -95,6 +96,15 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit } // do insert + uint32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + while (n < pSubmitBlk->nData) { + n += tGetTSRow(p + n, &row.tsRow); + + ASSERT(n <= pSubmitBlk->nData); + + // TODO + } return code; From 73cbc6f717cbf3815ec00fc34bedac041af90cd6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 12:53:23 +0000 Subject: [PATCH 16/37] feat: vnode multi-version --- source/dnode/vnode/src/inc/tsdb.h | 21 +++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 139 +++++++++++++------- 2 files changed, 114 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4f22ce768f..a62b4c4409 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -36,6 +36,8 @@ typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; typedef struct SDelOp SDelOp; +static int tsdbKeyCmprFn(const void *p1, const void *p2); + // tsdbMemTable2.c ============================================================================================== typedef struct SMemTable SMemTable; @@ -873,6 +875,25 @@ struct SDelOp { SDelOp *pNext; }; +static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { + TSDBKEY *pKey1 = (TSDBKEY *)p1; + TSDBKEY *pKey2 = (TSDBKEY *)p2; + + if (pKey1->ts < pKey2->ts) { + return -1; + } else if (pKey1->ts > pKey2->ts) { + return 1; + } + + if (pKey1->version < pKey2->version) { + return -1; + } else if (pKey1->version > pKey2->version) { + return 1; + } + + return 0; +} + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index aab190b0bb..cbc9fef686 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,19 +15,33 @@ #include "tsdb.h" -typedef struct SMemData SMemData; -// typedef struct SMemSkipList SMemSkipList; -// typedef struct SMemSkipListNode SMemSkipListNode; -// typedef struct SMemSkipListCurosr SMemSkipListCurosr; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; + +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[0]; +}; + +struct SMemSkipList { + uint32_t seed; + int32_t size; + int8_t maxLevel; + int8_t level; + SMemSkipListNode *pHead; + SMemSkipListNode *pTail; +}; struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SDelOp *delOpHead; - SDelOp *delOpTail; + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SDelOp *delOpHead; + SDelOp *delOpTail; + SMemSkipList sl; }; struct SMemTable { @@ -39,8 +53,17 @@ struct SMemTable { SArray *pArray; // SArray }; +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); +static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); +static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -96,16 +119,41 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit } // do insert - uint32_t n = 0; - uint8_t *p = pSubmitBlk->pData; + int32_t nt; + uint8_t *pt; + int32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + SVBufPool *pPool = pTsdb->pVnode->inUse; + int8_t level; + SMemSkipListNode *pNode; while (n < pSubmitBlk->nData) { - n += tGetTSRow(p + n, &row.tsRow); + nt = tGetTSRow(p + n, &row.tsRow); + n += nt; ASSERT(n <= pSubmitBlk->nData); - // TODO + // build the node + level = tsdbMemSkipListRandLevel(&pMemData->sl); + pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + nt + sizeof(version)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pNode->level = level; + tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), &row); + + // put the node (todo) + + pMemData->nRows++; + + // set info + if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row; + if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row; } + if (tsdbKeyCmprFn(&pMemTable->minKey, &pMemData->minKey) < 0) pMemTable->minKey = pMemData->minKey; + if (tsdbKeyCmprFn(&pMemTable->maxKey, &pMemData->maxKey) > 0) pMemTable->maxKey = pMemData->maxKey; + return code; _err: @@ -224,23 +272,40 @@ static int memDataPCmprFn(const void *p1, const void *p2) { return 0; } +static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = 0; + + n += tPutI64(p ? p + n : p, pRow->version); + n += tPutTSRow(p ? p + n : p, &pRow->tsRow); + + return n; +} + +static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = 0; + + n += tGetI64(p + n, &pRow->version); + n += tGetTSRow(p + n, &pRow->tsRow); + + return n; +} + +static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { + int8_t level = 1; + int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); + const uint32_t factor = 4; + + while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { + level++; + } + + return level; +} + #if 0 //==================================================================================== #define SL_MAX_LEVEL 5 -struct SMemSkipListNode { - int8_t level; - SMemSkipListNode *forwards[1]; // Windows does not allow 0 -}; - -struct SMemSkipList { - uint32_t seed; - int8_t maxLevel; - int8_t level; - int32_t size; - SMemSkipListNode pHead[1]; // Windows does not allow 0 -}; - struct SMemSkipListCurosr { SMemSkipList *pSl; SMemSkipListNode *pNodes[SL_MAX_LEVEL]; @@ -254,12 +319,6 @@ typedef struct { #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) -#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) -#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) -#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) - #define SL_HEAD_NODE(sl) ((sl)->pHead) #define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) #define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l) @@ -390,18 +449,6 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { - int8_t level = 1; - int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - const uint32_t factor = 4; - - while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { - level++; - } - - return level; -} - static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) { if (tEncodeI64(pEncoder, pRow->version) < 0) return -1; if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1; From 874a29d3eba2a43994f997a1be61015dad8af8c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 13:14:30 +0000 Subject: [PATCH 17/37] feat: vnode multi-version --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index cbc9fef686..94c88a14ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,6 @@ struct SMemData { tb_uid_t uid; TSDBKEY minKey; TSDBKEY maxKey; - int64_t nRows; SDelOp *delOpHead; SDelOp *delOpTail; SMemSkipList sl; @@ -59,6 +58,9 @@ struct SMemTable { #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) #define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) +#define SL_HEAD_FORWARD(sl, l) SL_NODE_FORWARD((sl)->pHead, l) +#define SL_TAIL_BACKWARD(sl, l) SL_NODE_FORWARD((sl)->pTail, l) + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); @@ -144,8 +146,6 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit // put the node (todo) - pMemData->nRows++; - // set info if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row; if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row; @@ -217,6 +217,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid}; SMemData *pMemData = NULL; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; // get idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); @@ -226,7 +227,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui } // create - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2); if (pMemData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -235,8 +236,18 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui pMemData->uid = uid; pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; - pMemData->nRows = 0; pMemData->delOpHead = pMemData->delOpTail = NULL; + pMemData->sl.seed = taosRand(); + pMemData->sl.size = 0; + pMemData->sl.maxLevel = maxLevel; + pMemData->sl.level = 0; + pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1]; + pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_HALF_SIZE(maxLevel)); + + for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { + SL_HEAD_FORWARD(&pMemData->sl, iLevel) = pMemData->sl.pTail; + SL_TAIL_BACKWARD(&pMemData->sl, iLevel) = pMemData->sl.pHead; + } if (idx < 0) idx = 0; if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { From 646357f1dcdf3c583d820cfd13de8998c9d5914c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 03:08:46 +0000 Subject: [PATCH 18/37] more --- include/common/tdataformat.h | 24 ++--- source/common/src/tdataformat.c | 182 ++++++++++++++++++++++++++++++-- 2 files changed, 186 insertions(+), 20 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index c75c5b41f2..b142c36b35 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,11 +48,11 @@ void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); -void tTSRowFree(STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder @@ -123,17 +123,17 @@ struct STSRowBuilder { struct SValue { union { - int8_t i8; - uint8_t u8; - int16_t i16; - uint16_t u16; - int32_t i32; - uint32_t u32; - int64_t i64; - uint64_t u64; - TSKEY ts; - float f; - double d; + int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT + uint8_t u8; // TSDB_DATA_TYPE_UTINYINT + int16_t i16; // TSDB_DATA_TYPE_SMALLINT + uint16_t u16; // TSDB_DATA_TYPE_USMALLINT + int32_t i32; // TSDB_DATA_TYPE_INT + uint32_t u32; // TSDB_DATA_TYPE_UINT + int64_t i64; // TSDB_DATA_TYPE_BIGINT + uint64_t u64; // TSDB_DATA_TYPE_UBIGINT + TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP + float f; // TSDB_DATA_TYPE_FLOAT + double d; // TSDB_DATA_TYPE_DOUBLE struct { uint32_t nData; uint8_t *pData; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 2994411693..69f859bb3b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -21,15 +21,10 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); -typedef struct SKVIdx { - int32_t cid; - int32_t offset; -} SKVIdx; - #pragma pack(push, 1) typedef struct { int16_t nCols; - SKVIdx idx[]; + uint8_t idx[]; } STSKVRow; #pragma pack(pop) @@ -44,7 +39,7 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); // SValue -static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -95,7 +90,7 @@ static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -187,6 +182,177 @@ static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu } // STSRow2 ======================================================================== +static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { + int32_t nColVal = taosArrayGetSize(pArray); + STColumn *pTColumn; + SColVal *pColVal; + + ASSERT(nColVal > 0); + + pRow->sver = pTSchema->version; + + // ts + pTColumn = &pTSchema->columns[0]; + pColVal = (SColVal *)taosArrayGet(pArray, 0); + + ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + pRow->ts = pColVal->value.ts; + + // other fields + int32_t iColVal = 1; + int32_t bidx; + uint32_t nv = 0; + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t flags = 0; + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + bidx = iColumn - 1; + pTColumn = &pTSchema->columns[iColumn]; + + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + if (pColVal->isNone) { + goto _set_none; + } else if (pColVal->isNull) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + flags |= TSROW_HAS_NONE; + // SET_BIT2(pb, bidx, 0); (todo) + continue; + + _set_null: + flags != TSROW_HAS_NULL; + // SET_BIT2(pb, bidx, 1); (todo) + continue; + + _set_value: + flags != TSROW_HAS_VAL; + // SET_BIT2(pb, bidx, 2); (todo) + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); + } else { + tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); + } + continue; + } + + ASSERT(flags); + switch (flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + pRow->nData = 0; + break; + case TSROW_HAS_VAL: + pRow->nData = pTSchema->flen + nv; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1); + break; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + pRow->nData = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; + break; + default: + break; + } +} + +static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { + int32_t nColVal = taosArrayGetSize(pArray); + STColumn *pTColumn; + SColVal *pColVal; + + ASSERT(nColVal > 0); + + pRow->sver = pTSchema->version; + + // ts + pTColumn = &pTSchema->columns[0]; + pColVal = (SColVal *)taosArrayGet(pArray, 0); + + ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + pRow->ts = pColVal->value.ts; + + // other fields + int32_t iColVal = 1; + uint32_t nv = 0; + uint8_t *pv = NULL; + uint8_t *pidx = NULL; + uint8_t flags = 0; + int16_t nCol = 0; + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + if (pColVal->isNone) { + goto _set_none; + } else if (pColVal->isNull) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + flags |= TSROW_HAS_NONE; + continue; + + _set_null: + flags != TSROW_HAS_NULL; + pidx[nCol++] = nv; + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + continue; + + _set_value: + flags != TSROW_HAS_VAL; + pidx[nCol++] = nv; + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + continue; + } +} + int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; // TODO From be2bfd513ef43b23db05776ba6cda1e6e95c8fc0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 03:16:09 +0000 Subject: [PATCH 19/37] more --- source/common/src/tdataformat.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 69f859bb3b..81328aba1b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -351,11 +351,33 @@ static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; } + + if (nv <= UINT8_MAX) { + // small + } else if (nv <= UINT16_MAX) { + // mid + } else { + // large + } } +// try-decide-build int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; - // TODO + STSRow2 rowT = {0}; + STSRow2 rowM = {0}; + + // try + tTupleTSRowNew(pArray, pTSchema, &rowT); + tMapTSRowNew(pArray, pTSchema, &rowM); + + // decide & build + if (rowT.nData <= rowM.nData) { + tTupleTSRowNew(pArray, pTSchema, &rowT); + } else { + tMapTSRowNew(pArray, pTSchema, &rowM); + } + return code; } From c388099efbc1704c0c9989845e7f11ff9359af89 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 13:37:21 +0800 Subject: [PATCH 20/37] refactor: adjust trans conflict dbname --- source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 8 ++++---- source/dnode/mnode/impl/src/mndSma.c | 4 ++-- source/dnode/mnode/impl/src/mndStb.c | 6 +++--- source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- source/dnode/mnode/impl/test/trans/trans2.cpp | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 9b063fb44f..6d1f371083 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -61,7 +61,7 @@ int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); -void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); +void mndTransSetDbName(STrans *pTrans, const char *dbname); void mndTransSetSerial(STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5d79708109..a9e6b018db 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -388,7 +388,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { } terrno = 0; - return TSDB_CODE_SUCCESS; + return terrno; } static void mndSetDefaultDbCfg(SDbCfg *pCfg) { @@ -550,7 +550,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - mndTransSetDbInfo(pTrans, &dbObj); + mndTransSetDbName(pTrans, dbObj.name); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; @@ -780,7 +780,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); - mndTransSetDbInfo(pTrans, pOld); + mndTransSetDbName(pTrans, pOld->name); if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; @@ -1040,7 +1040,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 7f81a99fd2..afde6d1a8c 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -506,7 +506,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); mndTransSetSerial(pTrans); if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; @@ -751,7 +751,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9ca7613519..acb344c8a5 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -754,7 +754,7 @@ _OVER: } int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; @@ -1261,7 +1261,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; @@ -1407,7 +1407,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bad513a89d..e191bb9b2a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -619,8 +619,8 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void * pTrans->paramLen = paramLen; } -void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) { - memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN); +void mndTransSetDbName(STrans *pTrans, const char *dbname) { + memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN); } void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index d518db2d38..022c82c73d 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -128,7 +128,7 @@ class MndTestTrans2 : public ::testing::Test { mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1); if (pDb != NULL) { - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); } int32_t code = mndTransPrepare(pMnode, pTrans); @@ -201,7 +201,7 @@ class MndTestTrans2 : public ::testing::Test { } if (pDb != NULL) { - mndTransSetDbInfo(pTrans, pDb); + mndTransSetDbName(pTrans, pDb->name); } int32_t code = mndTransPrepare(pMnode, pTrans); From 6aece12755ec889a1a553ff17f7c80dcaeb51f01 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 1 Jun 2022 14:39:40 +0800 Subject: [PATCH 21/37] fix parser stack overflow issue --- source/libs/parser/inc/sql.y | 4 + source/libs/parser/src/parser.c | 49 +- source/libs/parser/src/sql.c | 1451 ++++++++++++++++++++----------- 3 files changed, 976 insertions(+), 528 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 75eceedb1b..6c090a0790 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -15,11 +15,15 @@ #include #include +#define ALLOW_FORBID_FUNC + #include "functionMgt.h" #include "nodes.h" #include "parToken.h" #include "ttokendef.h" #include "parAst.h" + +#define YYSTACKDEPTH 0 } %syntax_error { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index bb70458f98..c2e1eba472 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -76,28 +76,8 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; + switch (pParam->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - pVal->datum.b = *((bool*)pParam->buffer); - break; - case TSDB_DATA_TYPE_TINYINT: - pVal->datum.i = *((int8_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_SMALLINT: - pVal->datum.i = *((int16_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_INT: - pVal->datum.i = *((int32_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_BIGINT: - pVal->datum.i = *((int64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_FLOAT: - pVal->datum.d = *((float*)pParam->buffer); - break; - case TSDB_DATA_TYPE_DOUBLE: - pVal->datum.d = *((double*)pParam->buffer); - break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); @@ -124,28 +104,13 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { pVal->node.resType.bytes = output + VARSTR_HEADER_SIZE; break; } - case TSDB_DATA_TYPE_TIMESTAMP: - pVal->datum.i = *((int64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UTINYINT: - pVal->datum.u = *((uint8_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_USMALLINT: - pVal->datum.u = *((uint16_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UINT: - pVal->datum.u = *((uint32_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UBIGINT: - pVal->datum.u = *((uint64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_MEDIUMBLOB: - // todo - default: + default: { + int32_t code = nodesSetValueNodeValue(pVal, pParam->buffer); + if (code) { + return code; + } break; + } } pVal->translate = true; return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 7fb89bdd7c..ff4fe4032e 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -32,11 +32,15 @@ #include #include +#define ALLOW_FORBID_FUNC + #include "functionMgt.h" #include "nodes.h" #include "parToken.h" #include "ttokendef.h" #include "parAst.h" + +#define YYSTACKDEPTH 0 /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -136,6 +140,7 @@ typedef union { #define YYFALLBACK 1 #define YYNSTATE 612 #define YYNRULE 451 +#define YYNRULE_WITH_ACTION 451 #define YYNTOKEN 237 #define YY_MAX_SHIFT 611 #define YY_MIN_SHIFTREDUCE 898 @@ -640,7 +645,31 @@ static const YYCODETYPE yy_lookahead[] = { /* 2090 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, /* 2100 */ 328, 329, 330, 357, 332, 357, 357, 335, 357, 357, /* 2110 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, - /* 2120 */ 348, 357, 357, 357, 352, + /* 2120 */ 348, 357, 357, 357, 352, 237, 237, 237, 237, 237, + /* 2130 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2140 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2150 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2160 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2170 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2180 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2190 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2200 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2210 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2220 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2230 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2240 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2250 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2260 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2270 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2280 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2290 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2300 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2310 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2320 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2330 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2340 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2350 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 2360 */ 237, 237, }; #define YY_SHIFT_COUNT (611) #define YY_SHIFT_MIN (0) @@ -2382,15 +2411,18 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -2405,16 +2437,8 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -2428,6 +2452,7 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && iyytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -4406,9 +4886,9 @@ static YYACTIONTYPE yy_reduce( break; /********** End reduce actions ************************************************/ }; - assert( yyruleno Date: Wed, 1 Jun 2022 07:10:33 +0000 Subject: [PATCH 22/37] feat: data format --- include/common/tdataformat.h | 17 +- source/common/src/tdataformat.c | 363 +++++++++++++++++++------------- 2 files changed, 218 insertions(+), 162 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b142c36b35..f72ad49416 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -41,19 +41,14 @@ typedef struct STag STag; int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); -// SColVal -// #define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) -// #define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) -// #define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) - // STSRow2 int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); void tTSRowFree(STSRow2 *pRow); - +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); -int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder #if 0 @@ -98,7 +93,9 @@ struct STSchema { #define TSROW_HAS_NONE ((uint8_t)0x1) #define TSROW_HAS_NULL ((uint8_t)0x2U) #define TSROW_HAS_VAL ((uint8_t)0x4U) -#define TSROW_KV_ROW ((uint8_t)0x10U) +#define TSROW_KV_SMALL ((uint8_t)0x10U) +#define TSROW_KV_MID ((uint8_t)0x20U) +#define TSROW_KV_BIG ((uint8_t)0x40U) struct STSRow2 { TSKEY ts; uint8_t flags; @@ -142,9 +139,9 @@ struct SValue { }; struct SColVal { + int16_t cid; int8_t isNone; int8_t isNull; - int16_t cid; SValue value; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 81328aba1b..28015f6bd4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -162,25 +162,6 @@ static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu return n; } -static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { - int32_t n = 0; - - memset(pColVal, 0, sizeof(*pColVal)); - if (isTuple) { - n += tGetValue(p + n, &pColVal->value, type); - } else { - n += tGetI16v(p + n, &pColVal->cid); - if (pColVal->cid < 0) { - pColVal->isNull = 1; - pColVal->cid = -pColVal->cid; - } else { - n += tGetValue(p + n, &pColVal->value, type); - } - } - - return n; -} - // STSRow2 ======================================================================== static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { int32_t nColVal = taosArrayGetSize(pArray); @@ -381,6 +362,206 @@ int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { return code; } +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { + int32_t code = 0; + + (*ppRow) = (STSRow2 *)taosMemoryMalloc(sizeof(**ppRow)); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + **ppRow = *pRow; + (*ppRow)->pData = NULL; + + if (pRow->nData) { + (*ppRow)->pData = taosMemoryMalloc(pRow->nData); + if ((*ppRow)->pData == NULL) { + taosMemoryFree(*ppRow); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy((*ppRow)->pData, pRow->pData, pRow->nData); + } + +_exit: + return code; +} + +void tTSRowFree(STSRow2 *pRow) { + if (pRow) { + if (pRow->pData) taosMemoryFree(pRow->pData); + taosMemoryFree(pRow); + } +} + +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint8_t isTuple = (pRow->flags & 0xf0 == 0) ? 1 : 0; + STColumn *pTColumn = &pTSchema->columns[iCol]; + uint8_t flags = pRow->flags & (uint8_t)0xf; + SValue value; + + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(flags); + ASSERT(pRow->sver == pTSchema->version); + + if (iCol == 0) { + value.ts = pRow->ts; + goto _return_value; + } + + if (flags == TSROW_HAS_NONE) { + *pColVal = (SColVal){.isNone = 1, .cid = pTColumn->colId}; + return; + } else if (flags == TSROW_HAS_NONE) { + *pColVal = (SColVal){.isNull = 1, .cid = pTColumn->colId}; + return; + } + + ASSERT(pRow->nData && pRow->pData); + + if (isTuple) { + uint8_t *pb = pRow->pData; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t *p; + uint8_t b; + + // bit + switch (flags) { + case TSROW_HAS_VAL: + pf = pb; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + goto _return_null; + } + case TSROW_HAS_VAL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_null; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT2(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else if (b == 1) { + goto _return_null; + } else { + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + break; + } + default: + ASSERT(0); + } + + ASSERT(pf); + + p = pf + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pv = pf + pTSchema->flen; + p = pv + *(VarDataOffsetT *)p; + } + tGetValue(p, &value, pTColumn->type); + goto _return_value; + } else { + STSKVRow *pRowK = (STSKVRow *)pRow->pData; + int16_t lidx = 0; + int16_t ridx = pRowK->nCols - 1; + uint8_t *p; + int16_t midx; + uint32_t n; + int16_t cid; + + ASSERT(pRowK->nCols > 0); + + if (pRow->flags & TSROW_KV_SMALL) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_MID) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_BIG) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + } else { + ASSERT(0); + } + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; + + if (pRow->flags & TSROW_KV_SMALL) { + n = ((uint8_t *)pRowK->idx)[midx]; + } else if (pRow->flags & TSROW_KV_MID) { + n = ((uint16_t *)pRowK->idx)[midx]; + } else { + n = ((uint32_t *)pRowK->idx)[midx]; + } + + n += tGetI16v(p + n, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + goto _return_null; + } else { + n += tGetValue(p + n, &value, pTColumn->type); + goto _return_value; + } + + return; + } else if (TABS(cid) > pTColumn->colId) { + ridx = midx - 1; + } else { + lidx = midx + 1; + } + } + + // not found, return NONE + goto _return_none; + } + +_return_none: + *pColVal = (SColVal){.cid = pTColumn->colId, .isNone = 1}; + return; + +_return_null: + *pColVal = (SColVal){.cid = pTColumn->colId, .isNull = 1}; + return; + +_return_value: + *pColVal = (SColVal){.cid = pTColumn->colId, .value = value}; + return; +} + +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { + int32_t code = 0; + SColVal cv; + + (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (*ppArray == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + tTSRowGet(pRow, pTSchema, iColumn, &cv); + taosArrayPush(*ppArray, &cv); + } + +_exit: + return code; +} + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; @@ -393,8 +574,11 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: + ASSERT(pRow->nData == 0); + ASSERT(pRow->pData == NULL); break; default: + ASSERT(pRow->nData && pRow->pData); n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData); break; } @@ -404,151 +588,26 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; - uint8_t flags; - n += tGetI64(p + n, pRow ? &pRow->ts : NULL); - n += tGetI8(p + n, pRow ? &pRow->flags : &flags); - n += tGetI32v(p + n, pRow ? &pRow->sver : NULL); + n += tGetI64(p + n, &pRow->ts); + n += tGetI8(p + n, &pRow->flags); + n += tGetI32v(p + n, &pRow->sver); - if (pRow) flags = pRow->flags; - switch (flags & 0xf) { + ASSERT(pRow->flags); + switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: + pRow->nData = 0; + pRow->pData = NULL; break; default: - n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL); + n += tGetBinary(p + n, &pRow->pData, &pRow->nData); break; } return n; } -int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) { - (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData); - if (*ppRow == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - (*ppRow)->ts = pRow->ts; - (*ppRow)->flags = pRow->flags; - (*ppRow)->sver = pRow->sver; - (*ppRow)->nData = pRow->nData; - if (pRow->nData) { - (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]); - memcpy((*ppRow)->pData, pRow->pData, pRow->nData); - } else { - (*ppRow)->pData = NULL; - } - - return 0; -} - -void tTSRowFree(STSRow2 *pRow) { - if (pRow) taosMemoryFree(pRow); -} - -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { -#if 0 - uint32_t n; - uint8_t *p; - uint8_t v; - int32_t bidx = iCol - 1; - STColumn *pTColumn = &pTSchema->columns[iCol]; - STSKVRow *pTSKVRow; - SKVIdx *pKVIdx; - - ASSERT(iCol != 0); - ASSERT(pTColumn->colId != 0); - - ASSERT(pRow->flags & 0xf != 0); - switch (pRow->flags & 0xf) { - case TSROW_HAS_NONE: - *pColVal = ColValNONE; - return 0; - case TSROW_HAS_NULL: - *pColVal = ColValNULL; - return 0; - } - - if (TSROW_IS_KV_ROW(pRow)) { - ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL); - - pTSKVRow = (STSKVRow *)pRow->pData; - pKVIdx = - bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); - if (pKVIdx == NULL) { - *pColVal = ColValNONE; - } else if (pKVIdx->offset < 0) { - *pColVal = ColValNULL; - } else { - p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; - pColVal->type = COL_VAL_DATA; - tGetBinary(p, &pColVal->pData, &pColVal->nData); - } - } else { - // get bitmap - p = pRow->pData; - switch (pRow->flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - v = GET_BIT1(p, bidx); - if (v == 0) { - *pColVal = ColValNONE; - } else { - *pColVal = ColValNULL; - } - return 0; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - v = GET_BIT1(p, bidx); - if (v == 1) { - p = p + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } else { - *pColVal = ColValNONE; - return 0; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = GET_BIT1(p, bidx); - if (v == 1) { - p = p + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } else { - *pColVal = ColValNULL; - return 0; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - v = GET_BIT2(p, bidx); - if (v == 0) { - *pColVal = ColValNONE; - return 0; - } else if (v == 1) { - *pColVal = ColValNULL; - return 0; - } else if (v == 2) { - p = p + BIT2_SIZE(pTSchema->numOfCols - 1); - break; - } else { - ASSERT(0); - } - default: - break; - } - - // get real value - p = p + pTColumn->offset; - pColVal->type = COL_VAL_DATA; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData); - } else { - pColVal->pData = p; - pColVal->nData = pTColumn->bytes; - } - } - -#endif - return 0; -} - // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); From 62e01aba2870d14707a0393da40d18ad2c970e2b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 07:16:18 +0000 Subject: [PATCH 23/37] refact: more --- source/common/src/tdataformat.c | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 28015f6bd4..a8c263dd7d 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -141,27 +141,6 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -// SColVal -static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { - int32_t n = 0; - - ASSERT(pColVal->isNone == 0); - if (isTuple) { - ASSERT(pColVal->isNull == 0); - n += tPutValue(p ? p + n : p, &pColVal->value, type); - } else { - if (pColVal->isNull) { - // -cid means NULL - n += tPutI16v(p ? p + n : p, -pColVal->cid); - } else { - n += tPutI16v(p ? p + n : p, pColVal->cid); - n += tPutValue(p ? p + n : p, &pColVal->value, type); - } - } - - return n; -} - // STSRow2 ======================================================================== static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { int32_t nColVal = taosArrayGetSize(pArray); @@ -232,9 +211,9 @@ static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { flags != TSROW_HAS_VAL; // SET_BIT2(pb, bidx, 2); (todo) if (IS_VAR_DATA_TYPE(pTColumn->type)) { - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); } else { - tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); + // tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); } continue; } @@ -323,13 +302,13 @@ static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { _set_null: flags != TSROW_HAS_NULL; pidx[nCol++] = nv; - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; _set_value: flags != TSROW_HAS_VAL; pidx[nCol++] = nv; - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; } From c2d95cbc0d1e47dbdaceedeecbbc29bad6c085aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 07:28:57 +0000 Subject: [PATCH 24/37] more --- include/common/tdataformat.h | 5 ++++- source/common/src/tdataformat.c | 12 +++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f72ad49416..cc311228b7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -42,7 +42,10 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 -int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +#define COL_VAL_NONE(CID) ((SColVal){.cid = (CID), .isNone = 1}) +#define COL_VAL_NULL(CID) ((SColVal){.cid = (CID), .isNull = 1}) +#define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .value = (V)}) + int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); void tTSRowFree(STSRow2 *pRow); void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a8c263dd7d..4dccdcb17e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -389,11 +389,9 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal } if (flags == TSROW_HAS_NONE) { - *pColVal = (SColVal){.isNone = 1, .cid = pTColumn->colId}; - return; + goto _return_none; } else if (flags == TSROW_HAS_NONE) { - *pColVal = (SColVal){.isNull = 1, .cid = pTColumn->colId}; - return; + goto _return_null; } ASSERT(pRow->nData && pRow->pData); @@ -510,15 +508,15 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal } _return_none: - *pColVal = (SColVal){.cid = pTColumn->colId, .isNone = 1}; + *pColVal = COL_VAL_NONE(pTColumn->colId); return; _return_null: - *pColVal = (SColVal){.cid = pTColumn->colId, .isNull = 1}; + *pColVal = COL_VAL_NULL(pTColumn->colId); return; _return_value: - *pColVal = (SColVal){.cid = pTColumn->colId, .value = value}; + *pColVal = COL_VAL_VALUE(pTColumn->colId, value); return; } From 26fab7d00be58c6901cdcf8dfbb57a931dc7d33f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 15:57:33 +0800 Subject: [PATCH 25/37] docs: correct a few missing changes --- docs-cn/12-taos-sql/07-function.md | 2 +- docs-en/12-taos-sql/07-function.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-cn/12-taos-sql/07-function.md b/docs-cn/12-taos-sql/07-function.md index 04d4adb7d4..7674967f09 100644 --- a/docs-cn/12-taos-sql/07-function.md +++ b/docs-cn/12-taos-sql/07-function.md @@ -668,7 +668,7 @@ SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] SELECT MODE(field_name) FROM tb_name [WHERE clause]; ``` -**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。不能匹配标签、时间戳输出。 +**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。 **返回数据类型**：同应用的字段。 diff --git a/docs-en/12-taos-sql/07-function.md b/docs-en/12-taos-sql/07-function.md index 4eaf7c8a68..74f5ad1de1 100644 --- a/docs-en/12-taos-sql/07-function.md +++ b/docs-en/12-taos-sql/07-function.md @@ -281,7 +281,7 @@ SELECT CONCAT(str1|column1, str2|column2, ...) FROM { tb_name | stb_name } [WHER **Return value type**: If all input strings are VARCHAR type, the result is VARCHAR type too. If any one of input strings is NCHAR type, then the result is NCHAR. -**Applicable data types**: VARCHAR, NCHAR. Can't be used on tag columns. At least 2 input strings are requird, and at most 8 input strings are allowed. +**Applicable data types**: VARCHAR, NCHAR. At least 2 input strings are requird, and at most 8 input strings are allowed. **Applicable table types**: table, STable From 7342aae0658e17acacb483403cd5d1a54e93ee61 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 1 Jun 2022 16:03:21 +0800 Subject: [PATCH 26/37] docs: correct a few missing changes --- docs-en/12-taos-sql/07-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-en/12-taos-sql/07-function.md b/docs-en/12-taos-sql/07-function.md index 74f5ad1de1..129b7eb0c3 100644 --- a/docs-en/12-taos-sql/07-function.md +++ b/docs-en/12-taos-sql/07-function.md @@ -297,7 +297,7 @@ SELECT CONCAT_WS(separator, str1|column1, str2|column2, ...) FROM { tb_name | st **Return value type**: If all input strings are VARCHAR type, the result is VARCHAR type too. If any one of input strings is NCHAR type, then the result is NCHAR. -**Applicable data types**: VARCHAR, NCHAR. Can't be used on tag columns. At least 3 input strings are requird, and at most 9 input strings are allowed. +**Applicable data types**: VARCHAR, NCHAR. At least 3 input strings are requird, and at most 9 input strings are allowed. **Applicable table types**: table, STable From 7e421c0092eefd04e00f54e13572ea0abb502b61 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:04:06 +0000 Subject: [PATCH 27/37] make it compile --- include/util/tencode.h | 187 ++++++++++++++++++++++++++++++++++------- 1 file changed, 157 insertions(+), 30 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index bde2185b02..3e1d675870 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -461,64 +461,151 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { } // =========================================== -#define tPutV(p, v) \ - int32_t n = 0; \ - for (;;) { \ - if (v <= 0x7f) { \ - if (p) p[n] = v; \ - n++; \ - break; \ - } \ - if (p) p[n] = (v & 0x7f) | 0x80; \ - n++; \ - v >>= 7; \ - } \ - return n; +#define tPutV(p, v) \ + do { \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; \ + } while (0) -#define tGetV(p, v) \ - int32_t n = 0; \ - if (v) *v = 0; \ - for (;;) { \ - if (p[n] <= 0x7f) { \ - if (v) (*v) |= (p[n] << (7 * n)); \ - n++; \ - break; \ - } \ - if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ - n++; \ - } \ - return n; +#define tGetV(p, v) \ + do { \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; \ + } while (0) // PUT +static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { + if (p) ((uint8_t*)p)[0] = v; + return sizeof(uint8_t); +} + static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { if (p) ((int8_t*)p)[0] = v; return sizeof(int8_t); } +static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { + if (p) ((uint16_t*)p)[0] = v; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { + if (p) ((int16_t*)p)[0] = v; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { + if (p) ((uint32_t*)p)[0] = v; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { + if (p) ((int32_t*)p)[0] = v; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { + if (p) ((uint64_t*)p)[0] = v; + return sizeof(uint64_t); +} + static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { if (p) ((int64_t*)p)[0] = v; return sizeof(int64_t); } -static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v) } +static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { + union { + uint32_t ui; + float f; + } v = {.f = f}; + + return tPutU32(p, v.ui); +} + +static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { + union { + uint64_t ui; + double d; + } v = {.d = d}; + + return tPutU64(p, v.ui); +} + +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } -static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v) } +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } +static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } + +// GET +static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { + if (v) *v = ((uint8_t*)p)[0]; + return sizeof(uint8_t); +} + static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { if (v) *v = ((int8_t*)p)[0]; return sizeof(int8_t); } +static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { + if (v) *v = ((uint16_t*)p)[0]; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { + if (v) *v = ((int16_t*)p)[0]; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { + if (v) *v = ((uint32_t*)p)[0]; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { + if (v) *v = ((int32_t*)p)[0]; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { + if (v) *v = ((uint64_t*)p)[0]; + return sizeof(uint64_t); +} + static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { if (v) *v = ((int64_t*)p)[0]; return sizeof(int64_t); } -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v) } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { int32_t n; @@ -530,7 +617,7 @@ static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { return n; } -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v) } +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { int32_t n; @@ -542,6 +629,46 @@ static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { return n; } +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { + int32_t n; + uint64_t tv; + + n = tGetU64v(p, &tv); + if (v) *v = ZIGZAGD(int64_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { + int32_t n = 0; + + union { + uint32_t ui; + float f; + } v; + + n = tGetU32(p, &v.ui); + + *f = v.f; + return n; +} + +static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { + int32_t n = 0; + + union { + uint64_t ui; + double d; + } v; + + n = tGetU64(p, &v.ui); + + *d = v.d; + return n; +} + // ===================== static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { int n = 0; From d84a5af792fef8428a228063c443c52d85576cdb Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 16:18:47 +0800 Subject: [PATCH 28/37] fix(os): remote fqdn error --- tests/pytest/util/dnodes.py | 1 + tests/system-test/0-others/taosShell.py | 6 ++++++ tests/system-test/0-others/taosShellError.py | 6 ++++++ tests/system-test/0-others/taosShellNetChk.py | 8 +++++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2e11b93e5f..21d235ee5c 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -397,6 +397,7 @@ class TDDnode: def stop(self): if (not self.remoteIP == ""): self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index) + tdLog.info("stop dnode%d"%self.index) return if self.valgrind == 0: toBeKilled = "taosd" diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 9c8cd85b46..89a820544e 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -84,6 +84,12 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() + if (not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP serverPort = '7080' rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index e00fe89461..a30c6d85fb 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -86,6 +86,12 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() + if (not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP serverPort = '7080' rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index c81d4af3c5..0446f7e1fd 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -86,6 +86,12 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() + if (not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP ) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP serverPort = '7080' rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} @@ -196,7 +202,7 @@ class TDTestCase: pktNum = '10' role = 'client' if platform.system().lower() == 'windows': - taosCmd = buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c'] + taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c'] taosCmd = taosCmd.replace('\\','\\\\') else: taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] From 10042e9a5f3cd1f6e09f398af10f230052037cfa Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 16:20:20 +0800 Subject: [PATCH 29/37] fix(os): remote fqdn error --- source/libs/scalar/test/scalar/CMakeLists.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 672cb5a3de..86b936d93a 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -17,9 +17,7 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) -if(NOT TD_WINDOWS) - add_test( - NAME scalarTest - COMMAND scalarTest - ) -endif(NOT TD_WINDOWS) +add_test( + NAME scalarTest + COMMAND scalarTest +) From 8c53f98184cd7a686a43bd03c888068ef9b3a1c1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:31:54 +0000 Subject: [PATCH 30/37] make it pass windows comile --- include/common/tmsg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f38a0f8551..83ea44cd86 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -945,7 +945,6 @@ typedef struct { int64_t timeInFetchQueue; } SQnodeLoad; - typedef struct { int32_t sver; // software version int64_t dnodeVer; // dnode table version in sdb @@ -1977,7 +1976,7 @@ typedef struct { int8_t killConnection; int8_t align[3]; SEpSet epSet; - SArray *pQnodeList; + SArray* pQnodeList; } SQueryHbRspBasic; typedef struct { @@ -2676,6 +2675,7 @@ typedef struct { } SVDeleteReq; typedef struct { + int32_t code; // TODO } SVDeleteRsp; From 6f94998e062475950dcb851be37f622f8b42d8b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:34:03 +0000 Subject: [PATCH 31/37] make it pass windows compile --- include/util/tencode.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 3e1d675870..a13afd4448 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -538,7 +538,8 @@ static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { union { uint32_t ui; float f; - } v = {.f = f}; + } v; + v.f = f; return tPutU32(p, v.ui); } @@ -547,7 +548,8 @@ static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { union { uint64_t ui; double d; - } v = {.d = d}; + } v; + v.d = d; return tPutU64(p, v.ui); } From fc3e1a329242c621b405165f8ec9ea509c01123d Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 1 Jun 2022 16:43:09 +0800 Subject: [PATCH 32/37] Update 13-operators.md --- docs-cn/12-taos-sql/13-operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-cn/12-taos-sql/13-operators.md b/docs-cn/12-taos-sql/13-operators.md index 1ffc823044..22b78455fb 100644 --- a/docs-cn/12-taos-sql/13-operators.md +++ b/docs-cn/12-taos-sql/13-operators.md @@ -35,8 +35,8 @@ TDengine 支持 `UNION ALL` 和 `UNION` 操作符。UNION ALL 将查询返回的 | --- | :---------------: | -------------------------------------------------------------------- | -------------------- | | 1 | = | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 相等 | | 2 | <\>, != | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 不相等 | -| 3 | \>, \< | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于，小于 | -| 4 | \>=, \<= | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于等于，小于等于 | +| 3 | \>, < | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于，小于 | +| 4 | \>=, <= | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于等于，小于等于 | | 5 | IS [NOT] NULL | 所有类型 | 是否为空值 | | 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 闭区间比较 | | 7 | IN | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值相等 | From 41132d779a05adbe42bbbb6225c2772a9888b2b9 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 1 Jun 2022 16:43:23 +0800 Subject: [PATCH 33/37] Update 13-operators.md --- docs-en/12-taos-sql/13-operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-en/12-taos-sql/13-operators.md b/docs-en/12-taos-sql/13-operators.md index e393c82c76..0ca9ec4943 100644 --- a/docs-en/12-taos-sql/13-operators.md +++ b/docs-en/12-taos-sql/13-operators.md @@ -35,8 +35,8 @@ TDengine provides 2 set operators: `UNION ALL` and `UNION`. `UNION ALL` combines | --- | :---------------: | ------------------------------------------------------------------- | ----------------------------------------------- | | 1 | = | Except for BLOB, MEDIUMBLOB and JSON | Equal | | 2 | <\>, != | Except for BLOB, MEDIUMBLOB, JSON and primary key of timestamp type | Not equal | -| 3 | \>, \< | Except for BLOB, MEDIUMBLOB and JSON | Greater than, less than | -| 4 | \>=, \<= | Except for BLOB, MEDIUMBLOB and JSON | Greater than or equal to, less than or equal to | +| 3 | \>, < | Except for BLOB, MEDIUMBLOB and JSON | Greater than, less than | +| 4 | \>=, <= | Except for BLOB, MEDIUMBLOB and JSON | Greater than or equal to, less than or equal to | | 5 | IS [NOT] NULL | Any types | Is NULL or NOT | | 6 | [NOT] BETWEEN AND | Except for BLOB, MEDIUMBLOB and JSON | In a value range or not | | 7 | IN | Except for BLOB, MEDIUMBLOB, JSON and primary key of timestamp type | In a list of values or not | From 8b4d1a4962c4b0abee3924c52fbcf8d38d426d2e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 1 Jun 2022 16:50:57 +0800 Subject: [PATCH 34/37] fix(os): remote fqdn error --- tests/system-test/0-others/taosShell.py | 2 +- tests/system-test/0-others/taosShellError.py | 2 +- tests/system-test/0-others/taosShellNetChk.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 89a820544e..046db93c49 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -84,7 +84,7 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() - if (not tdDnodes.dnodes[0].remoteIP == ""): + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): try: config = eval(tdDnodes.dnodes[0].remoteIP) hostname = config["host"] diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index a30c6d85fb..2369e4d580 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -86,7 +86,7 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() - if (not tdDnodes.dnodes[0].remoteIP == ""): + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): try: config = eval(tdDnodes.dnodes[0].remoteIP) hostname = config["host"] diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 0446f7e1fd..3c99ddb8d6 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -86,7 +86,7 @@ class TDTestCase: #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ # 'serverPort': 7080, 'firstEp': 'trd02:7080'} hostname = socket.gethostname() - if (not tdDnodes.dnodes[0].remoteIP == ""): + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): try: config = eval(tdDnodes.dnodes[0].remoteIP ) hostname = config["host"] From 7d58761e1dbc00bdc5c88639e323674f74cbfbc8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 1 Jun 2022 17:04:45 +0800 Subject: [PATCH 35/37] chore: merge two example directories to be one (#13358) [TD16198] --- example/CMakeLists.txt | 49 -------------------------- examples/c/CMakeLists.txt | 50 +++++++++++++++++++++++++++ {example/src => examples/c}/demoapi.c | 0 {example/src => examples/c}/tmq.c | 0 {example/src => examples/c}/tstream.c | 0 5 files changed, 50 insertions(+), 49 deletions(-) delete mode 100644 example/CMakeLists.txt rename {example/src => examples/c}/demoapi.c (100%) rename {example/src => examples/c}/tmq.c (100%) rename {example/src => examples/c}/tstream.c (100%) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt deleted file mode 100644 index 365b1b7172..0000000000 --- a/example/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -add_executable(tmq "") -add_executable(tstream "") -add_executable(demoapi "") - -target_sources(tmq - PRIVATE - "src/tmq.c" -) - -target_sources(tstream - PRIVATE - "src/tstream.c" -) - -target_sources(demoapi - PRIVATE - "src/demoapi.c" -) - -target_link_libraries(tmq - taos_static -) - -target_link_libraries(tstream - taos_static -) - -target_link_libraries(demoapi - taos_static -) - -target_include_directories(tmq - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_include_directories(tstream - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_include_directories(demoapi - PUBLIC "${TD_SOURCE_DIR}/include/client" - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) -SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream) -SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi) diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 8bc86abbb1..eff492945e 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -11,6 +11,56 @@ IF (TD_LINUX) #TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread ) #ADD_EXECUTABLE(epoll epoll.c) #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) + + add_executable(tmq "") + add_executable(tstream "") + add_executable(demoapi "") + + target_sources(tmq + PRIVATE + "tmq.c" + ) + + target_sources(tstream + PRIVATE + "tstream.c" + ) + + target_sources(demoapi + PRIVATE + "demoapi.c" + ) + + target_link_libraries(tmq + taos_static + ) + + target_link_libraries(tstream + taos_static + ) + + target_link_libraries(demoapi + taos_static + ) + + target_include_directories(tmq + PUBLIC "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + ) + + target_include_directories(tstream + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + ) + + target_include_directories(demoapi + PUBLIC "${TD_SOURCE_DIR}/include/client" + PUBLIC "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + ) + + SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) + SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream) + SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) diff --git a/example/src/demoapi.c b/examples/c/demoapi.c similarity index 100% rename from example/src/demoapi.c rename to examples/c/demoapi.c diff --git a/example/src/tmq.c b/examples/c/tmq.c similarity index 100% rename from example/src/tmq.c rename to examples/c/tmq.c diff --git a/example/src/tstream.c b/examples/c/tstream.c similarity index 100% rename from example/src/tstream.c rename to examples/c/tstream.c From e1a5000e50f13628cccae9fb24dcc2432818179c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 1 Jun 2022 17:26:15 +0800 Subject: [PATCH 36/37] refactor(tmq): tq sink and push --- include/common/tmsg.h | 5 +- include/libs/stream/tstream.h | 13 +- source/common/src/tmsg.c | 5 +- source/dnode/vnode/CMakeLists.txt | 5 +- source/dnode/vnode/src/inc/sma.h | 4 +- source/dnode/vnode/src/inc/tq.h | 56 +++--- source/dnode/vnode/src/inc/vnodeInt.h | 6 +- source/dnode/vnode/src/sma/sma.c | 3 +- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- source/dnode/vnode/src/tq/tq.c | 216 +--------------------- source/dnode/vnode/src/tq/tqCommit.c | 5 + source/dnode/vnode/src/tq/tqMeta.c | 26 +++ source/dnode/vnode/src/tq/tqPush.c | 180 ++++++++++++++++++ source/dnode/vnode/src/tq/tqRead.c | 35 ++++ source/dnode/vnode/src/tq/tqSink.c | 34 ++++ source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/libs/stream/src/tstream.c | 11 +- 17 files changed, 341 insertions(+), 267 deletions(-) create mode 100644 source/dnode/vnode/src/tq/tqSink.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a679cc11cd..c4b67ec3a3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -244,7 +244,7 @@ typedef struct { const void* pMsg; } SSubmitMsgIter; -int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); @@ -945,7 +945,6 @@ typedef struct { int64_t timeInFetchQueue; } SQnodeLoad; - typedef struct { int32_t sver; // software version int64_t dnodeVer; // dnode table version in sdb @@ -1968,7 +1967,7 @@ typedef struct { int8_t killConnection; int8_t align[3]; SEpSet epSet; - SArray *pQnodeList; + SArray* pQnodeList; } SQueryHbRspBasic; typedef struct { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8aaf9a79dc..0525cbf367 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -61,11 +61,8 @@ enum { }; typedef struct { - int8_t type; - - int32_t sourceVg; - int64_t sourceVer; - + int8_t type; + int64_t ver; int32_t* dataRef; SSubmitReq* data; } SStreamDataSubmit; @@ -111,6 +108,8 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) } } +SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); + int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput); void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput); @@ -209,8 +208,6 @@ struct SStreamTask { int32_t nodeId; SEpSet epSet; - // source preprocess - // exec STaskExec exec; @@ -318,8 +315,6 @@ int32_t streamDequeueOutput(SStreamTask* pTask, void** output); int32_t streamTaskRun(SStreamTask* pTask); -int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); - int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 233623c616..f542b73ae3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,7 +28,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -165,7 +165,6 @@ int32_t tDecodeSQueryNodeLoad(SDecoder *pDecoder, SQueryNodeLoad *pLoad) { return 0; } - int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pEp->inUse); @@ -3053,7 +3052,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; - if(pReq->isTsma) { + if (pReq->isTsma) { taosMemoryFreeClear(pReq->pTsma); } return 0; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index ea2a256663..9ff6b59fc1 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -55,8 +55,9 @@ target_sources( "src/tq/tqMeta.c" "src/tq/tqRead.c" "src/tq/tqOffset.c" - #"src/tq/tqPush.c" - #"src/tq/tqCommit.c" + "src/tq/tqPush.c" + "src/tq/tqSink.c" + "src/tq/tqCommit.c" ) target_include_directories( vnode diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 0601df61e7..03dd2ea66c 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -219,7 +219,7 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk void *tdFreeRSmaInfo(SRSmaInfo *pInfo); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version); +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version); // TODO: This is the basic params, and should wrap the params to a queryHandle. int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); @@ -227,4 +227,4 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query } #endif -#endif /*_TD_VNODE_SMA_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_SMA_H_*/ diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 89ea969d92..56d86c26a0 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -66,33 +66,27 @@ struct STqReadHandle { // tqPush typedef struct { - int64_t consumerId; - int32_t epoch; - int32_t skipLogNum; - int64_t reqOffset; - SRpcHandleInfo info; - SRWLatch lock; -} STqPushHandle; + STaosQueue* queue; + STaosQall* qall; + void* qItem; +} STqInputQ; -#if 0 typedef struct { - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + // msg info int64_t consumerId; + int64_t reqOffset; + int64_t processedVer; int32_t epoch; - int8_t subType; - // int8_t withTbName; - // int8_t withSchema; - // int8_t withTag; - char* qmsg; - SHashObj* pDropTbUid; - STqPushHandle pushHandle; - // SRWLatch lock; - SWalReadHandle* pWalReader; - // task number should be the same with fetch thread - STqReadHandle* pExecReader[5]; - qTaskInfo_t task[5]; -} STqExec; -#endif + int32_t skipLogNum; + // rpc info + int64_t reqId; + SRpcHandleInfo rpcInfo; + // exec + int8_t inputStatus; + int8_t execStatus; + STqInputQ inputQ; + SRWLatch lock; +} STqPushHandle; // tqExec @@ -154,27 +148,21 @@ typedef struct { static STqMgmt tqMgmt = {0}; -// init once -int tqInit(); -void tqCleanUp(); - -// int32_t tEncodeSTqExec(SEncoder* pEncoder, const STqExec* pExec); -// int32_t tDecodeSTqExec(SDecoder* pDecoder, STqExec* pExec); - -int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); -int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); - +// tqRead int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum); +// tqExec int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); // tqMeta - int32_t tqMetaOpen(STQ* pTq); int32_t tqMetaClose(STQ* pTq); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); +// tqSink +void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); + // tqOffset STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); void STqOffsetClose(STqOffsetStore*); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e3a0c94ccc..2d72981f00 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -125,6 +125,8 @@ int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader); int32_t tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData); // tq +int tqInit(); +void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); @@ -145,11 +147,11 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); -int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version); +int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); -int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq* pReq); +int32_t tdProcessRSmaCreate(SVnode* pVnode, SVCreateStbReq* pReq); int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index 0e7ce385a1..7a2b6a2757 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -15,7 +15,6 @@ #include "sma.h" - // TODO: Who is responsible for resource allocate and release? int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { int32_t code = TSDB_CODE_SUCCESS; @@ -37,7 +36,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { return code; } -int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version) { +int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f771e73c8a..f88afcaddf 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -932,7 +932,7 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde * @param msg SSubmitReq * @return int32_t */ -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) { +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { // no time-range-sma, just return success if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 93f305ba77..172caf8724 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,169 +81,10 @@ void tqClose(STQ* pTq) { // TODO } -int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1; - if (tEncodeI64(pEncoder, pHandle->consumerId) < 0) return -1; - if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1; - if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1; - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; - } - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pHandle->subKey) < 0) return -1; - if (tDecodeI64(pDecoder, &pHandle->consumerId) < 0) return -1; - if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1; - if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1; - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; - } - tEndDecode(pDecoder); - return 0; -} - -int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pTq->handles, pIter); - if (pIter == NULL) break; - STqHandle* pExec = (STqHandle*)pIter; - if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - for (int32_t i = 0; i < 5; i++) { - int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd); - ASSERT(code == 0); - } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { - if (!isAdd) { - int32_t sz = taosArrayGetSize(tbUidList); - for (int32_t i = 0; i < sz; i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); - } - } - } else { - // tq update id - } - } - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { - int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); - ASSERT(code == 0); - } - } - return 0; -} - -int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) { - if (msgType != TDMT_VND_SUBMIT) return 0; - void* pIter = NULL; - STqHandle* pHandle = NULL; - SSubmitReq* pReq = (SSubmitReq*)msg; - int32_t workerId = 4; - int64_t fetchOffset = ver; - - while (1) { - pIter = taosHashIterate(pTq->pushMgr, pIter); - if (pIter == NULL) break; - pHandle = *(STqHandle**)pIter; - - taosWLockLatch(&pHandle->pushHandle.lock); - - SMqDataBlkRsp rsp = {0}; - rsp.reqOffset = pHandle->pushHandle.reqOffset; - rsp.blockData = taosArrayInit(0, sizeof(void*)); - rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); - - if (msgType == TDMT_VND_SUBMIT) { - tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); - } else { - // TODO - ASSERT(0); - } - - if (rsp.blockNum == 0) { - taosWUnLockLatch(&pHandle->pushHandle.lock); - continue; - } - - ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); - ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); - - rsp.rspOffset = fetchOffset; - - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - // todo free - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pHandle->pushHandle.epoch; - ((SMqRspHead*)buf)->consumerId = pHandle->pushHandle.consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqDataBlkRsp(&abuf, &rsp); - - SRpcMsg resp = { - .info = pHandle->pushHandle.info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&resp); - - memset(&pHandle->pushHandle.info, 0, sizeof(SRpcHandleInfo)); - taosWUnLockLatch(&pHandle->pushHandle.lock); - - tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum, - rsp.reqOffset, rsp.rspOffset); - - // TODO destroy - taosArrayDestroy(rsp.blockData); - taosArrayDestroy(rsp.blockDataLen); - } - - return 0; -} - -int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - if (msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; - - if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { - // TODO handle sma error - } - void* data = taosMemoryMalloc(msgLen); - if (data == NULL) { - return -1; - } - memcpy(data, msg, msgLen); - - tqProcessStreamTrigger(pTq, data); - } - - return 0; -} - -int tqCommit(STQ* pTq) { - // do nothing - return 0; -} - int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t waitTime = pReq->timeout; + int64_t timeout = pReq->timeout; int32_t reqEpoch = pReq->epoch; int64_t fetchOffset; @@ -286,20 +127,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { rsp.withSchema = false; - rsp.withTag = false; } else { rsp.withSchema = true; - rsp.blockSchema = taosArrayInit(0, sizeof(void*)); - rsp.withTag = false; + rsp.blockSchema = taosArrayInit(0, sizeof(void*)); } while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { - tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); + tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", + consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } @@ -310,27 +149,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SWalReadHead* pHead = &pHeadWithCkSum->head; -#if 0 - // add to pushMgr - taosWLockLatch(&pExec->pushHandle.lock); - - pExec->pushHandle.consumerId = consumerId; - pExec->pushHandle.epoch = reqEpoch; - pExec->pushHandle.reqOffset = rsp.reqOffset; - pExec->pushHandle.skipLogNum = rsp.skipLogNum; - pExec->pushHandle.handle = pMsg; - - taosWUnLockLatch(&pExec->pushHandle.lock); - - // TODO add timer - - // TODO: the pointer will always be valid? - taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*)); - taosArrayDestroy(rsp.blockData); - taosArrayDestroy(rsp.blockDataLen); - return 0; -#endif - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); @@ -471,24 +289,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - - ASSERT(pTask->tbSink.pTSchema); - SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, - pTask->tbSink.stbFullName, pVnode->config.vgId); - /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ - // build write msg - SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = pReq, - .contLen = ntohl(pReq->length), - }; - - ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0); -} - int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { @@ -579,9 +379,11 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { continue; } - streamDataSubmitRefInc(pSubmit); - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); + if (pSubmitClone == NULL) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + continue; + } taosWriteQitem(pTask->inputQ, pSubmitClone); int8_t execStatus = atomic_load_8(&pTask->status); diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index e31566f3fa..7b116bff2e 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -14,3 +14,8 @@ */ #include "tq.h" + +int tqCommit(STQ* pTq) { + // do nothing + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 74162a9f49..9447c4007b 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -15,6 +15,32 @@ #include "tdbInt.h" #include "tq.h" +static int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1; + if (tEncodeI64(pEncoder, pHandle->consumerId) < 0) return -1; + if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1; + if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1; + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; + } + tEndEncode(pEncoder); + return pEncoder->pos; +} + +static int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pHandle->subKey) < 0) return -1; + if (tDecodeI64(pDecoder, &pHandle->consumerId) < 0) return -1; + if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1; + if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1; + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1; + } + tEndDecode(pDecoder); + return 0; +} + int tqExecKeyCompare(const void* pKey1, int32_t kLen1, const void* pKey2, int32_t kLen2) { return strcmp(pKey1, pKey2); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e31566f3fa..f23a14472c 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -14,3 +14,183 @@ */ #include "tq.h" + +int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { + // 1. guard and set status executing + // 2. check processedVer + // 2.1. if not missed, get msg from queue + // 2.2. if missed, scan wal + // + // 3. exec, after each success, update processed ver + // first run + // set exec status closing + // second run + // set exec status idle + // + // 4. if get result + // 4.1 set exec input status blocked and exec status idle + // 4.2 rpc send + // 4.3 clear rpc info + return 0; +} + +int32_t tqOpenPushHandle(STQ* pTq, STqHandle* pHandle) { + memset(&pHandle->pushHandle, 0, sizeof(STqPushHandle)); + pHandle->pushHandle.inputQ.queue = taosOpenQueue(); + pHandle->pushHandle.inputQ.qall = taosAllocateQall(); + if (pHandle->pushHandle.inputQ.queue == NULL || pHandle->pushHandle.inputQ.qall == NULL) { + if (pHandle->pushHandle.inputQ.queue) { + taosCloseQueue(pHandle->pushHandle.inputQ.queue); + } + if (pHandle->pushHandle.inputQ.qall) { + taosFreeQall(pHandle->pushHandle.inputQ.qall); + } + return -1; + } + return 0; +} + +void tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHandleInfo* pInfo, int64_t processedVer) { + memcpy(&pHandle->pushHandle.rpcInfo, pInfo, sizeof(SRpcHandleInfo)); + atomic_store_64(&pHandle->pushHandle.reqId, reqId); + atomic_store_64(&pHandle->pushHandle.processedVer, processedVer); + atomic_store_8(&pHandle->pushHandle.inputStatus, TASK_INPUT_STATUS__NORMAL); + // set timeout timer +} + +int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) { + int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); + if (pSubmitClone == NULL) { + return -1; + } + taosWriteQitem(pHandle->pushHandle.inputQ.queue, pSubmitClone); + return 0; + } + return -1; +} + +int32_t tqSendExecReq(STQ* pTq, STqHandle* pHandle) { + // + return 0; +} + +int32_t tqEnqueueAll(STQ* pTq, SSubmitReq* pReq) { + void* pIter = NULL; + SStreamDataSubmit* pSubmit = streamDataSubmitNew(pReq); + if (pSubmit == NULL) { + return -1; + } + + while (1) { + pIter = taosHashIterate(pTq->handles, pIter); + if (pIter == NULL) break; + STqHandle* pHandle = (STqHandle*)pIter; + if (tqEnqueue(pHandle, pSubmit) < 0) { + continue; + } + int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus); + if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + tqSendExecReq(pTq, pHandle); + } + } + + streamDataSubmitRefDec(pSubmit); + + return 0; +} + +int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) { + if (msgType != TDMT_VND_SUBMIT) return 0; + void* pIter = NULL; + STqHandle* pHandle = NULL; + SSubmitReq* pReq = (SSubmitReq*)msg; + int32_t workerId = 4; + int64_t fetchOffset = ver; + + while (1) { + pIter = taosHashIterate(pTq->pushMgr, pIter); + if (pIter == NULL) break; + pHandle = *(STqHandle**)pIter; + + taosWLockLatch(&pHandle->pushHandle.lock); + + SMqDataBlkRsp rsp = {0}; + rsp.reqOffset = pHandle->pushHandle.reqOffset; + rsp.blockData = taosArrayInit(0, sizeof(void*)); + rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (msgType == TDMT_VND_SUBMIT) { + tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); + } else { + // TODO + ASSERT(0); + } + + if (rsp.blockNum == 0) { + taosWUnLockLatch(&pHandle->pushHandle.lock); + continue; + } + + ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); + ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); + + rsp.rspOffset = fetchOffset; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + // todo free + return -1; + } + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pHandle->pushHandle.epoch; + ((SMqRspHead*)buf)->consumerId = pHandle->pushHandle.consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqDataBlkRsp(&abuf, &rsp); + + SRpcMsg resp = { + .info = pHandle->pushHandle.rpcInfo, + .pCont = buf, + .contLen = tlen, + .code = 0, + }; + tmsgSendRsp(&resp); + + memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo)); + taosWUnLockLatch(&pHandle->pushHandle.lock); + + tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum, + rsp.reqOffset, rsp.rspOffset); + + // TODO destroy + taosArrayDestroy(rsp.blockData); + taosArrayDestroy(rsp.blockDataLen); + } + + return 0; +} + +int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { + if (msgType == TDMT_VND_SUBMIT) { + if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; + + if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { + // TODO handle sma error + } + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + return -1; + } + memcpy(data, msg, msgLen); + + tqProcessStreamTrigger(pTq, data); + } + + return 0; +} + diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 1f5d3b7f53..8909a00c72 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -298,3 +298,38 @@ int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) return 0; } + +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pTq->handles, pIter); + if (pIter == NULL) break; + STqHandle* pExec = (STqHandle*)pIter; + if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + for (int32_t i = 0; i < 5; i++) { + int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd); + ASSERT(code == 0); + } + } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { + if (!isAdd) { + int32_t sz = taosArrayGetSize(tbUidList); + for (int32_t i = 0; i < sz; i++) { + int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); + taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); + } + } + } else { + // tq update id + } + } + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); + ASSERT(code == 0); + } + } + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c new file mode 100644 index 0000000000..5c0bf971fb --- /dev/null +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pRes = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + + ASSERT(pTask->tbSink.pTSchema); + SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, + pTask->tbSink.stbFullName, pVnode->config.vgId); + /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ + // build write msg + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pReq, + .contLen = ntohl(pReq->length), + }; + + ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0); +} diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 178ef28e5d..f97367e989 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -189,4 +189,4 @@ void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } -void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; } \ No newline at end of file +void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; } diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index e033645667..7d406a7144 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -35,6 +35,16 @@ void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) { return (void*)buf; } +SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { + SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); + if (pSubmitClone == NULL) { + return NULL; + } + streamDataSubmitRefInc(pSubmit); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + return pSubmitClone; +} + static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { SStreamDispatchReq req = { .streamId = pTask->streamId, @@ -207,7 +217,6 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { if (pRes == NULL) return -1; while (1) { int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - void* exec = pTask->exec.executor; if (execStatus == TASK_STATUS__IDLE) { // first run, from qall, handle failure from last exec pRes = streamExecForQall(pTask, pRes); From 7fcfd45a03e6501a971544d9767c35f33b6a1e25 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 17:26:25 +0800 Subject: [PATCH 37/37] feat: increase the number of database replicas --- include/common/tmsg.h | 9 +- include/common/tmsgdef.h | 6 +- source/common/src/tmsg.c | 13 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 6 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 21 ++-- source/dnode/mnode/impl/inc/mndVgroup.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 49 ++++---- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 41 +++---- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/inc/vnd.h | 7 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 9 +- source/dnode/vnode/src/vnd/vnodeSync.c | 64 +++++++--- tests/script/tsim/db/alter_replica_13.sim | 124 ++++++++++++++++++++ 14 files changed, 261 insertions(+), 93 deletions(-) create mode 100644 tests/script/tsim/db/alter_replica_13.sim diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e0f6c403aa..a923e96ca5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1002,7 +1002,6 @@ typedef struct { typedef struct { int32_t vgId; - int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; int64_t dbUid; int32_t vgVersion; @@ -1025,15 +1024,13 @@ typedef struct { int8_t compression; int8_t strict; int8_t cacheLastRow; + int8_t isTsma; + int8_t standby; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention - - // for tsma - int8_t isTsma; - } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -1071,8 +1068,8 @@ typedef struct { int8_t walLevel; int8_t strict; int8_t cacheLastRow; - int8_t replica; int8_t selfIndex; + int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; } SAlterVnodeReq; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index d7db2399e8..8a53244117 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -221,9 +221,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "vnode-alter-config", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "vnode-alter-replica", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL) // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 350e0f066d..74cc9e9e51 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2934,7 +2934,6 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; - if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; @@ -2957,6 +2956,8 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->isTsma) < 0) return -1; + if (tEncodeI8(&encoder, pReq->standby) < 0) return -1; if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { @@ -2972,7 +2973,6 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } - if (tEncodeI8(&encoder, pReq->isTsma) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2986,7 +2986,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; @@ -3009,6 +3008,8 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->standby) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { @@ -3035,8 +3036,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * } } - if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; - tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -3123,8 +3122,8 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; - if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; @@ -3154,8 +3153,8 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 34576d5441..5c5316e3a3 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -219,9 +219,9 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8374db129f..9965ec50a2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -156,13 +156,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashEnd = pCreate->hashEnd; pCfg->hashMethod = pCreate->hashMethod; + pCfg->standby = pCfg->standby; pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.replicaNum = pCreate->replica; memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); for (int i = 0; i < pCreate->replica; ++i) { - pCfg->syncCfg.nodeInfo[i].nodePort = pCreate->replicas[i].port; - snprintf(pCfg->syncCfg.nodeInfo[i].nodeFqdn, sizeof(pCfg->syncCfg.nodeInfo[i].nodeFqdn), "%s", - pCreate->replicas[i].fqdn); + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + pNode->nodePort = pCreate->replicas[i].port; + tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); } } @@ -175,6 +176,8 @@ static void vmGenerateWrapperCfg(SVnodeMgmt *pMgmt, SCreateVnodeReq *pCreate, SW int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SCreateVnodeReq createReq = {0}; + SVnodeCfg vnodeCfg = {0}; + SWrapperCfg wrapperCfg = {0}; int32_t code = -1; char path[TSDB_FILENAME_LEN] = {0}; @@ -183,12 +186,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dDebug("vgId:%d, create vnode req is received, tsma:%d", createReq.vgId, createReq.isTsma); - - SVnodeCfg vnodeCfg = {0}; + dDebug("vgId:%d, create vnode req is received, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, + createReq.standby); vmGenerateVnodeCfg(&createReq, &vnodeCfg); - - SWrapperCfg wrapperCfg = {0}; vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); @@ -313,8 +313,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index c9099b6b05..3f4f3f2053 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -36,7 +36,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode); int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); -void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a9e6b018db..c062a2c552 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,7 +261,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -270,7 +270,7 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p mndReleaseDnode(pMnode, pDnode); int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby); if (pReq == NULL) return -1; action.pCont = pReq; @@ -286,7 +286,7 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -296,7 +296,7 @@ static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD action.pCont = pReq; action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; + action.msgType = msgType; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -467,7 +467,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) { + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) { return -1; } } @@ -688,29 +688,37 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + sdbFreeRaw(pRedoRaw); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); return 0; } static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pCommitRaw = mndDbActionEncode(pNew); if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbFreeRaw(pCommitRaw); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); return 0; } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) { + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { return -1; } } else { SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + mndTransSetSerial(pTrans); + if (newVgroup.replica < pDb->cfg.replications) { mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); @@ -720,9 +728,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); @@ -733,15 +741,18 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } return 0; @@ -774,18 +785,16 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * } static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) { - int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); - if (pTrans == NULL) goto _OVER; - + if (pTrans == NULL) return -1; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); + int32_t code = -1; mndTransSetDbName(pTrans, pOld->name); if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index afde6d1a8c..35b834f816 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -421,7 +421,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, // todo add sma info here int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, false); if (pReq == NULL) return -1; action.pCont = pReq; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 161fc5379c..d9d04a82d5 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -51,9 +51,10 @@ int32_t mndInitVgroup(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndProcessAlterVnodeRsp); mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_VNODE_RSP, mndProcessCompactVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndProcessCompactVnodeRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -188,10 +189,10 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { +void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, + bool standby) { SCreateVnodeReq createReq = {0}; createReq.vgId = pVgroup->vgId; - createReq.dnodeId = pDnode->id; memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN); createReq.dbUid = pDb->uid; createReq.vgVersion = pVgroup->version; @@ -218,6 +219,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; + createReq.standby = standby; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; @@ -274,7 +276,6 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ alterReq.strict = pDb->cfg.strict; alterReq.cacheLastRow = pDb->cfg.cacheLastRow; alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &alterReq.replicas[v]; @@ -290,13 +291,6 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ mndReleaseDnode(pMnode, pVgidDnode); } -#if 0 - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } -#endif - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); if (contLen < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -508,7 +502,7 @@ _OVER: taosArrayDestroy(pArray); return code; } - +//---> int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { @@ -536,7 +530,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pVgid->role = TAOS_SYNC_STATE_ERROR; pDnode->numOfVnodes++; mInfo("db:%s, vgId:%d, vnode_index:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); @@ -547,16 +541,15 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } - +//---> int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { - int32_t removedNum = 0; - taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } + int32_t removedNum = 0; for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { SDnodeObj *pDnode = taosArrayGet(pArray, d); @@ -662,6 +655,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; + int64_t curMs = taosGetTimestampMs(); SDbObj *pDb = NULL; if (strlen(pShow->db) > 0) { @@ -701,12 +695,15 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); + bool online = false; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId); + if (pDnode != NULL) { + online = mndIsDnodeOnline(pMnode, pDnode, curMs); + mndReleaseDnode(pMnode, pDnode); + } + char buf1[20] = {0}; - SDnodeObj *pDnodeObj = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId); - ASSERT(pDnodeObj != NULL); - bool isOffLine = !mndIsDnodeOnline(pMnode, pDnodeObj, taosGetTimestampMs()); - const char *role = isOffLine ? "OFFLINE" : syncStr(pVgroup->vnodeGid[i].role); - + const char *role = online ? syncStr(pVgroup->vnodeGid[i].role) : "OFFLINE"; STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9cfcf2b086..104b96d217 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -177,6 +177,7 @@ struct SVnodeCfg { uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; + int8_t standby; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index eb3382ac4c..a5907cf991 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -81,9 +81,10 @@ int32_t vnodeSyncCommit(SVnode* pVnode); int32_t vnodeAsyncCommit(SVnode* pVnode); // vnodeSync.c -int32_t vnodeSyncOpen(SVnode* pVnode, char* path); -void vnodeSyncStart(SVnode* pVnode); -void vnodeSyncClose(SVnode* pVnode); +int32_t vnodeSyncOpen(SVnode* pVnode, char* path); +void vnodeSyncStart(SVnode* pVnode); +void vnodeSyncClose(SVnode* pVnode); +void vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4b237bc703..c449691028 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -22,7 +22,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { SDecoder dc = {0}; @@ -88,6 +88,9 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { } } break; + case TDMT_VND_ALTER_REPLICA: { + vnodeSyncAlter(pVnode, pMsg); + } break; default: break; } @@ -154,7 +157,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pMsg->contLen - sizeof(SMsgHead)) < 0) { } } break; - case TDMT_VND_ALTER_VNODE: + case TDMT_VND_ALTER_CONFIG: break; default: ASSERT(0); @@ -784,7 +787,7 @@ _exit: return 0; } -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d146877853..8792fbbb0c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -27,6 +27,7 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { .vgId = pVnode->config.vgId, + .isStandBy = pVnode->config.standby, .syncCfg = pVnode->config.syncCfg, .pWal = pVnode->pWal, .msgcb = NULL, @@ -49,28 +50,72 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { return 0; } +void vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { + SAlterVnodeReq req = {0}; + if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + vError("vgId:%d, failed to alter replica since %s", TD_VID(pVnode), terrstr()); + SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + tmsgSendRsp(&rsp); + } + + vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica); + SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex}; + for (int32_t r = 0; r < req.replica; ++r) { + SNodeInfo *pNode = &cfg.nodeInfo[r]; + tstrncpy(pNode->nodeFqdn, req.replicas[r].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = req.replicas[r].port; + vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort); + } + + if (syncReconfig(pVnode->sync, &cfg) != 0) { + vError("vgId:%d, failed to propose sync reconfig since %s", TD_VID(pVnode), terrstr()); + SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + tmsgSendRsp(&rsp); + } +} + void vnodeSyncStart(SVnode *pVnode) { syncSetMsgCb(pVnode->sync, &pVnode->msgCb); - syncStart(pVnode->sync); + if (pVnode->config.standby) { + syncStartStandBy(pVnode->sync); + } else { + syncStart(pVnode->sync); + } } void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } -int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { +int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } return code; } -int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return tmsgSendReq(pEpSet, pMsg); } +int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { + int32_t code = tmsgSendReq(pEpSet, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { vnodeGetSnapshot(pFsm->data, pSnapshot); return 0; } +void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { + SVnode *pVnode = pFsm->data; + vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode)); + + // todo rpc response here +} + void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SyncIndex beginIndex = SYNC_INDEX_INVALID; if (pFsm->FpGetSnapshot != NULL) { @@ -87,20 +132,12 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SVnode *pVnode = (SVnode *)(pFsm->data); + SVnode *pVnode = pFsm->data; SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); SRpcMsg applyMsg; syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); syncApplyMsgDestroy(pSyncApplyMsg); - /* - SRpcMsg applyMsg; - applyMsg = *pMsg; - applyMsg.pCont = rpcMallocCont(applyMsg.contLen); - assert(applyMsg.contLen == pMsg->contLen); - memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen); - */ - // recover handle for response SRpcMsg saveRpcMsg; int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg); @@ -142,14 +179,13 @@ void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); - memset(pFsm, 0, sizeof(*pFsm)); pFsm->data = pVnode; pFsm->FpCommitCb = vnodeSyncCommitMsg; pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg; pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpGetSnapshot = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = NULL; - pFsm->FpReConfigCb = NULL; + pFsm->FpReConfigCb = vnodeSyncReconfig; return pFsm; } \ No newline at end of file diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim new file mode 100644 index 0000000000..8ab6eb64fd --- /dev/null +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -0,0 +1,124 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1: create dnodes +sql create dnode $hostname port 7200 + +$loop_cnt = 0 +step1: + $loop_cnt = $loop_cnt + 1 + sleep 1000 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create database +sql create database db vgroups 1 +sql show databases +if $rows != 3 then + return -1 +endi +if $data(db)[4] != 1 then + return -1 +endi + +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi + +# vnodes +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi + +# v1_dnode +sql show db.vgroups +if $data(2)[3] != 2 then + return -1 +endi + +sql_error alter database db replica 3 +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" +sql create table db.ctb using db.stb tags(101, "102") +sql insert into db.ctb values(now, 1, "2") +sql select * from db.stb +if $rows != 1 then + return -1 +endi + +print =============== step3: create dnodes +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +step3: + $loop_cnt = $loop_cnt + 1 + sleep 1000 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data24 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step3 +endi +if $data(2)[4] != ready then + goto step3 +endi +if $data(3)[4] != ready then + goto step3 +endi +if $data(4)[4] != ready then + goto step3 +endi + +return +print ============= step4: alter database +sql alter database db replica 3 +if $rows != 3 then + return -1 +endi +if $data(db)[4] != 3 then + return -1 +endi + +sql select * from db.stb +if $rows != 1 then + return -1 +endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT