From d68bfc972f6be757cd05fce46ecc5771f2440bb2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 01:40:18 +0000 Subject: [PATCH 01/25] refact: tsdb --- source/dnode/vnode/CMakeLists.txt | 2 - source/dnode/vnode/src/tsdb/tsdbSma.c | 2203 --------------------- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 128 -- 3 files changed, 2333 deletions(-) delete mode 100644 source/dnode/vnode/src/tsdb/tsdbSma.c delete mode 100644 source/dnode/vnode/src/tsdb/tsdbTDBImpl.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d988f97188..b80adb9a95 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -35,7 +35,6 @@ target_sources( "src/sma/smaTimeRange.c" # tsdb - # "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbFile.c" @@ -45,7 +44,6 @@ target_sources( "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - # "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" "src/tsdb/tsdbSnapshot.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c deleted file mode 100644 index 45b17a0180..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ /dev/null @@ -1,2203 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbSma.h" -#include "tsdb.h" - -static const char *TSDB_SMA_DNAME[] = { - "", // TSDB_SMA_TYPE_BLOCK - "tsma", // TSDB_SMA_TYPE_TIME_RANGE - "rsma", // TSDB_SMA_TYPE_ROLLUP -}; - -#undef _TEST_SMA_PRINT_DEBUG_LOG_ -#define SMA_STORAGE_TSDB_DAYS 30 -#define SMA_STORAGE_TSDB_TIMES 10 -#define SMA_STORAGE_SPLIT_HOURS 24 -#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 -#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds - -#define SMA_STATE_HASH_SLOT 4 -#define SMA_STATE_ITEM_HASH_SLOT 32 - -#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test -#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test - -typedef struct SRSmaInfo SRSmaInfo; -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - -typedef struct SPoolMem { - int64_t size; - struct SPoolMem *prev; - struct SPoolMem *next; -} SPoolMem; - -struct SSmaEnv { - TdThreadRwlock lock; - int8_t type; - TXN txn; - SPoolMem *pPool; - SDiskID did; - TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? - char *path; // relative path - SSmaStat *pStat; -}; - -#define SMA_ENV_LOCK(env) ((env)->lock) -#define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_DID(env) ((env)->did) -#define SMA_ENV_ENV(env) ((env)->dbEnv) -#define SMA_ENV_PATH(env) ((env)->path) -#define SMA_ENV_STAT(env) ((env)->pStat) -#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) - -typedef struct { - STsdb *pTsdb; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int32_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SDBFile dFile; - int32_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int8_t storageLevel; - int8_t days; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef struct { - /** - * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from - * Streaming Module or TSDB local persistence. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. - * - TSDB_SMA_STAT_DROPPED: 1)sma dropped - * N.B. only applicable to tsma - */ - int8_t state; // ETsdbSmaStat - SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma *pSma; // cache schema -} SSmaStatItem; - -#define RSMA_TASK_INFO_HASH_SLOT 8 -struct SRSmaInfo { - void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t -}; - -struct SSmaStat { - union { - SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; - }; - T_REF_DECLARE() -}; -#define SMA_STAT_ITEMS(s) ((s)->smaStatItems) -#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) - -static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { - // Note: free/kill may in RC - qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); - if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { - qDestroyTask(otaskHandle); - } -} - -static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { - if (pInfo->taskInfo[i]) { - tsdbFreeTaskHandle(pInfo->taskInfo[i]); - } - } - return NULL; -} - -// declaration of static functions - -// expired window -static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version); -static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version); -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); -static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); -static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv); -static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); -static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); - -// read data -// TODO: This is the basic params, and should wrap the params to a queryHandle. -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); - -// insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); -static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg); -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); - -static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); -static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, - int8_t level); -// mgmt interface -static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); - -// Pool Memory -static SPoolMem *openPool(); -static void clearPool(SPoolMem *pPool); -static void closePool(SPoolMem *pPool); -static void *poolMalloc(void *arg, size_t size); -static void poolFree(void *arg, void *ptr); - -static int tsdbSmaBeginCommit(SSmaEnv *pEnv); -static int tsdbSmaEndCommit(SSmaEnv *pEnv); - -// implementation -static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { - return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); -} -static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { - return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); -} - -static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockRdlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockWrlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockUnlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); - - pPool->prev = pPool->next = pPool; - pPool->size = 0; - - return pPool; -} - -static void clearPool(SPoolMem *pPool) { - if (!pPool) return; - - SPoolMem *pMem; - - do { - pMem = pPool->next; - - if (pMem == pPool) break; - - pMem->next->prev = pMem->prev; - pMem->prev->next = pMem->next; - pPool->size -= pMem->size; - - taosMemoryFree(pMem); - } while (1); - - assert(pPool->size == 0); -} - -static void closePool(SPoolMem *pPool) { - if (pPool) { - clearPool(pPool); - taosMemoryFree(pPool); - } -} - -static void *poolMalloc(void *arg, size_t size) { - void *ptr = NULL; - SPoolMem *pPool = (SPoolMem *)arg; - SPoolMem *pMem; - - pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); - if (!pMem) { - assert(0); - } - - pMem->size = sizeof(*pMem) + size; - pMem->next = pPool->next; - pMem->prev = pPool; - - pPool->next->prev = pMem; - pPool->next = pMem; - pPool->size += pMem->size; - - ptr = (void *)(&pMem[1]); - return ptr; -} - -static void poolFree(void *arg, void *ptr) { - SPoolMem *pPool = (SPoolMem *)arg; - SPoolMem *pMem; - - pMem = &(((SPoolMem *)ptr)[-1]); - - pMem->next->prev = pMem->prev; - pMem->prev->next = pMem->next; - pPool->size -= pMem->size; - - taosMemoryFree(pMem); -} - -int32_t tsdbInitSma(STsdb *pTsdb) { - // tSma - int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(REPO_META(pTsdb), false)); - if (numOfTSma > 0) { - atomic_store_16(&REPO_TSMA_NUM(pTsdb), (int16_t)numOfTSma); - } - // TODO: rSma - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { - if (pStatItem) { - return atomic_load_8(&pStatItem->state); - } - return TSDB_SMA_STAT_UNKNOWN; -} - -static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { - if (!pStatItem) { - return false; - } - - if (state) { - *state = atomic_load_8(&pStatItem->state); - return *state == TSDB_SMA_STAT_OK; - } - return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; -} - -static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; -} - -static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; -} - -static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); - } -} - -static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); - } -} - -static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); - } -} - -static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); -} - -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did) { - SSmaEnv *pEnv = NULL; - - pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); - if (!pEnv) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMA_ENV_TYPE(pEnv) = smaType; - - int code = taosThreadRwlockInit(&(pEnv->lock), NULL); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(pEnv); - return NULL; - } - - ASSERT(path && (strlen(path) > 0)); - SMA_ENV_PATH(pEnv) = strdup(path); - if (!SMA_ENV_PATH(pEnv)) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - SMA_ENV_DID(pEnv) = did; - - if (tsdbInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - char aname[TSDB_FILENAME_LEN] = {0}; - tfsAbsoluteName(REPO_TFS(pTsdb), did, path, aname); - if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - if (!(pEnv->pPool = openPool())) { - tsdbFreeSmaEnv(pEnv); - return NULL; - } - - return pEnv; -} - -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) { - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (!(*pEnv)) { - if (!(*pEnv = tsdbNewSmaEnv(pTsdb, smaType, path, did))) { - return TSDB_CODE_FAILED; - } - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Release resources allocated for its member fields, not including itself. - * - * @param pSmaEnv - * @return int32_t - */ -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { - if (pSmaEnv) { - tsdbDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); - taosMemoryFreeClear(pSmaEnv->pStat); - taosMemoryFreeClear(pSmaEnv->path); - taosThreadRwlockDestroy(&(pSmaEnv->lock)); - tsdbCloseDBEnv(pSmaEnv->dbEnv); - closePool(pSmaEnv->pPool); - } -} - -void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { - tsdbDestroySmaEnv(pSmaEnv); - taosMemoryFreeClear(pSmaEnv); - return NULL; -} - -static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_INC(pStat); - tsdbDebug("vgId:%d ref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); - return 0; -} - -static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_DEC(pStat); - tsdbDebug("vgId:%d unref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); - return 0; -} - -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { - ASSERT(pSmaStat != NULL); - - if (*pSmaStat) { // no lock - return TSDB_CODE_SUCCESS; - } - - /** - * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tsdbNew). - * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if - * tsdbInitSmaStat invoked in other multithread environment later. - */ - if (!(*pSmaStat)) { - *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); - if (!(*pSmaStat)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( - RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - - if (!SMA_STAT_INFO_HASH(*pSmaStat)) { - taosMemoryFreeClear(*pSmaStat); - return TSDB_CODE_FAILED; - } - } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - SMA_STAT_ITEMS(*pSmaStat) = - taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - - if (!SMA_STAT_ITEMS(*pSmaStat)) { - taosMemoryFreeClear(*pSmaStat); - return TSDB_CODE_FAILED; - } - } else { - ASSERT(0); - } - } - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (pItem) { - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - } - } - return pItem; -} - -static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { - if (pSmaStatItem) { - tdDestroyTSma(pSmaStatItem->pSma); - taosMemoryFreeClear(pSmaStatItem->pSma); - taosHashCleanup(pSmaStatItem->expiredWindows); - taosMemoryFreeClear(pSmaStatItem); - } - return NULL; -} - -/** - * @brief Release resources allocated for its member fields, not including itself. - * - * @param pSmaStat - * @return int32_t - */ -int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { - if (pSmaStat) { - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL); - while (item) { - SSmaStatItem *pItem = *(SSmaStatItem **)item; - tsdbFreeSmaStatItem(pItem); - item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item); - } - taosHashCleanup(SMA_STAT_ITEMS(pSmaStat)); - } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); - while (infoHash) { - SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; - tsdbFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); - } - taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); - } else { - ASSERT(0); - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { - SSmaEnv *pEnv = NULL; - - // return if already init - switch (smaType) { - case TSDB_SMA_TYPE_TIME_RANGE: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)))) { - return TSDB_CODE_SUCCESS; - } - break; - case TSDB_SMA_TYPE_ROLLUP: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)))) { - return TSDB_CODE_SUCCESS; - } - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - - // init sma env - tsdbLockRepo(pTsdb); - pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)) - : atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); - if (!pEnv) { - char rname[TSDB_FILENAME_LEN] = {0}; - - SDiskID did = {0}; - tfsAllocDisk(REPO_TFS(pTsdb), TFS_PRIMARY_LEVEL, &did); - if (did.level < 0 || did.id < 0) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); - - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rname, did) != TSDB_CODE_SUCCESS) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - - if (tsdbInitSmaEnv(pTsdb, smaType, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { - tsdbUnlockRepo(pTsdb); - return TSDB_CODE_FAILED; - } - - (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&REPO_TSMA_ENV(pTsdb), pEnv) - : atomic_store_ptr(&REPO_RSMA_ENV(pTsdb), pEnv); - } - tsdbUnlockRepo(pTsdb); - - return TSDB_CODE_SUCCESS; -}; - -static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(REPO_META(pTsdb), indexUid, true); - if (!pSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, - tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pSma = pSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - tsdbWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", REPO_ID(pTsdb), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", REPO_ID(pTsdb), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pTsdb - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) { - tsdbTrace("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb)); - return TSDB_CODE_SUCCESS; - } - - if (!REPO_META(pTsdb)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = REPO_TSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tsdbRefSmaStat(pTsdb, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - pSW = tdFreeTSmaWrapper(pSW); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - tdFreeTSmaWrapper(pSW); - break; - } - if (!pSW || (pTSma->tableUid != pBlock->suid)) { - if (pSW) { - pSW = tdFreeTSmaWrapper(pSW); - } - if (!(pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = REPO_CFG(pTsdb)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); - } else { - tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - REPO_ID(pTsdb), pTSma->indexUid, winSKey); - } - } - } - - tsdbUnRefSmaStat(pTsdb, pStat); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pTsdb - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", REPO_ID(pTsdb), - skey, indexUid); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", REPO_ID(pTsdb), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, REPO_ID(pTsdb), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Judge the tSma storage level - * - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { - // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? - switch (intervalUnit) { - case TIME_UNIT_HOUR: - if (interval < SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MINUTE: - if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_SECOND: - if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MILLISECOND: - if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_MICROSECOND: - if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - case TIME_UNIT_NANOSECOND: - if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { - return SMA_STORAGE_LEVEL_DFILESET; - } - break; - default: - break; - } - return SMA_STORAGE_LEVEL_TSDB; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = TSDB_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - tsdbCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - STsdb *pTsdb = pSmaH->pTsdb; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pTsdb - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel) { - STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb); - int32_t daysPerFile = pCfg->days; - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; - } - - return daysPerFile; -} - -static int tsdbSmaBeginCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - // start a new txn - tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { - tsdbWarn("tsdbSma tdb begin commit fail"); - return -1; - } - return 0; -} - -static int tsdbSmaEndCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - - // Commit current txn - if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { - tsdbWarn("tsdbSma tdb end commit fail"); - return -1; - } - tdbTxnClose(pTxn); - clearPool(pEnv->pPool); - return 0; -} - -/** - * @brief Insert/Update Time-range-wise SMA data. - * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. - * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. - * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The - * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). - * - The destination file of one data block for some interval is determined by its start TS key. - * - * @param pTsdb - * @param msg - * @return int32_t - */ -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - const SArray *pDataBlocks = (const SArray *)msg; - - // TODO: destroy SSDataBlocks(msg) - - // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus - // the sma data would arrive ahead of the update-expired-window msg. - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = REPO_TSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pSma = pItem->pSma; - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); - int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != TSDB_IVLD_FID) { - tsdbSmaEndCommit(pEnv); - tsdbCloseDBF(&tSmaH.dFile); - } - tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); - if (tsdbOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - tsdbSmaBeginCommit(pEnv); - } - - if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - tsdbWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); - tsdbSmaEndCommit(pEnv); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - tsdbDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - REPO_ID(pTsdb), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, pStat, indexUid, skey); - } else { - tsdbWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - REPO_ID(pTsdb), skey, tlen, indexUid); - } - - printf("\n"); - } - } - tsdbSmaEndCommit(pEnv); // TODO: not commit for every insert - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pTsdb - * @param msg - * @return int32_t - */ -static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)); - - // clear local cache - if (pEnv) { - tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tsdbSmaStatIsDropped(pItem)) { - tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tsdbWLockSma(pEnv); - if (tsdbSmaStatIsDropped(pItem)) { - tsdbUnLockSma(pEnv); - tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tsdbSmaStatSetDropped(pItem); - tsdbUnLockSma(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - tsdbDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal); - break; - } - tsdbDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - tsdbDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", REPO_ID(pTsdb), indexUid, nSleep, - refVal); - break; - }; - } - - tsdbFreeSmaStatItem(pItem); - tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) { - STsdb *pTsdb = pSmaH->pTsdb; - - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - const SArray *pDataBlocks = (const SArray *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); - int64_t indexUid = SMA_TEST_INDEX_UID; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tsdbRefSmaStat(pTsdb, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pSma = pItem->pSma; - - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); - int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); -#if 0 - int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); - if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { - tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - return TSDB_CODE_FAILED; - } - - if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { - tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - return TSDB_CODE_FAILED; - } - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); -#endif - - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSmaH - * @param pTsdb - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit) { - pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); - pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit); - pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - STsdb *pTsdb = pSmaH->pTsdb; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - int32_t nSmaFs = taosArrayGetSize(smaFs); - - tsdbCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief - * - * @param pTsdb Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tsdbRefSmaStat(pTsdb, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tsdbUnRefSmaStat(pTsdb, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tsdbUnRefSmaStat(pTsdb, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), - querySKey, indexUid); - } else { - tsdbDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey, - indexUid); - } - - STSma *pTSma = pItem->pSma; -#endif - - STSmaReadH tReadH = {0}; - tsdbInitTSmaReadH(&tReadH, pTsdb, pTSma->interval, pTSma->intervalUnit); - tsdbCloseDBF(&tReadH.dFile); - - tsdbUnRefSmaStat(pTsdb, pStat); - - tsdbInitTSmaFile(&tReadH, indexUid, querySKey); - if (tsdbOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 1; - tsdbEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), - tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - tsdbCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tsdbSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tsdbGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - tsdbCloseDBF(&tReadH.dFile); - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { - SSmaCfg vCreateSmaReq = {0}; - if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); - return -1; - } - - tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); - - // record current timezone of server side - vCreateSmaReq.tSma.timezoneInt = tsTimezone; - - if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { - // TODO: handle error - tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); - tdDestroyTSma(&vCreateSmaReq.tSma); - return -1; - } - - tsdbTSmaAdd(pTsdb, 1); - - tdDestroyTSma(&vCreateSmaReq.tSma); - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tdDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // - - if (metaDropTSma(REPO_META(pTsdb), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tsdbDropTSmaData(pTsdb, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tsdbTSmaSub(pTsdb, 1); - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. - * - * @param pTsdb - * @param pMeta - * @param pReq - * @return int32_t - */ -int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) { - if (!pReq->rollup) { - tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - SRSmaParam *param = &pReq->pRSmaParam; - - if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { - tsdbWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); - if (pRSmaInfo) { - tsdbWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); - return TSDB_CODE_SUCCESS; - } - - pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); - if (!pRSmaInfo) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); - if (!pReadHandle) { - taosMemoryFree(pRSmaInfo); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - SReadHandle handle = { - .reader = pReadHandle, - .meta = pMeta, - .pMsgCb = pMsgCb, - }; - - if (param->qmsg1) { - pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle); - if (!pRSmaInfo->taskInfo[0]) { - taosMemoryFree(pRSmaInfo); - taosMemoryFree(pReadHandle); - return TSDB_CODE_FAILED; - } - } - - if (param->qmsg2) { - pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle); - if (!pRSmaInfo->taskInfo[1]) { - taosMemoryFree(pRSmaInfo); - taosMemoryFree(pReadHandle); - return TSDB_CODE_FAILED; - } - } - - if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->suid); - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief store suid/[uids], prefer to use array and then hash - * - * @param pStore - * @param suid - * @param uid - * @return int32_t - */ -static int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { - // prefer to store suid/uids in array - if ((suid == pStore->suid) || (pStore->suid == 0)) { - if (pStore->suid == 0) { - pStore->suid = suid; - } - if (uid) { - if (!pStore->tbUids) { - if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - if (!taosArrayPush(pStore->tbUids, uid)) { - return TSDB_CODE_FAILED; - } - } - } else { - // store other suid/uids in hash when multiple stable/table included in 1 batch of request - if (!pStore->uidHash) { - pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if (!pStore->uidHash) { - return TSDB_CODE_FAILED; - } - } - if (uid) { - SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); - if (uidArray && ((uidArray = *(SArray **)uidArray))) { - taosArrayPush(uidArray, uid); - } else { - SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); - if (!pUidArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - if (!taosArrayPush(pUidArray, uid)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { - return TSDB_CODE_FAILED; - } - } - } else { - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { - return TSDB_CODE_FAILED; - } - } - } - return TSDB_CODE_SUCCESS; -} - -void tsdbUidStoreDestory(STbUidStore *pStore) { - if (pStore) { - if (pStore->uidHash) { - if (pStore->tbUids) { - // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys. - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { - SArray *arr = *(SArray **)pIter; - taosArrayDestroy(arr); - pIter = taosHashIterate(pStore->uidHash, pIter); - } - } - taosHashCleanup(pStore->uidHash); - } - taosArrayDestroy(pStore->tbUids); - } -} - -void *tsdbUidStoreFree(STbUidStore *pStore) { - if (pStore) { - tsdbUidStoreDestory(pStore); - taosMemoryFree(pStore); - } - return NULL; -} - -/** - * @brief fetch suid/uids when create child tables of rollup SMA - * - * @param pTsdb - * @param ppStore - * @param suid - * @param uid - * @return int32_t - */ -int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) { - SSmaEnv *pEnv = REPO_RSMA_ENV((STsdb *)pTsdb); - - // only applicable to rollup SMA ctables - if (!pEnv) { - return TSDB_CODE_SUCCESS; - } - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - return TSDB_CODE_FAILED; - } - - // info cached when create rsma stable and return directly for non-rsma ctables - if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) { - return TSDB_CODE_SUCCESS; - } - - ASSERT(ppStore != NULL); - - if (!(*ppStore)) { - if (tsdbUidStoreInit(ppStore) != 0) { - return TSDB_CODE_FAILED; - } - } - - if (tsdbUidStorePut(*ppStore, suid, &uid) != 0) { - *ppStore = tsdbUidStoreFree(*ppStore); - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - if (!suid || !tbUids) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64, REPO_ID(pTsdb), *suid); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - return TSDB_CODE_FAILED; - } - - if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { - tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), - pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - - if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { - tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), - pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { - if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { - return TSDB_CODE_SUCCESS; - } - - if (tsdbUpdateTbUidListImpl(pTsdb, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; - } - - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { - tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - SArray *pTbUids = *(SArray **)pIter; - - if (tsdbUpdateTbUidListImpl(pTsdb, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { - taosHashCancelIterate(pStore->uidHash, pIter); - return TSDB_CODE_FAILED; - } - - pIter = taosHashIterate(pStore->uidHash, pIter); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - - terrno = TSDB_CODE_SUCCESS; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - - if (!pBlock) break; - tsdbUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, - int8_t level) { - SArray *pResult = NULL; - - if (!taskInfo) { - tsdbDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, REPO_ID(pTsdb), level, suid); - return TSDB_CODE_SUCCESS; - } - - tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, - suid); - - qSetStreamInput(taskInfo, pMsg, inputType, false); - while (1) { - SSDataBlock *output = NULL; - uint64_t ts; - if (qExecTask(taskInfo, &output, &ts) < 0) { - ASSERT(false); - } - if (!output) { - break; - } - if (!pResult) { - pResult = taosArrayInit(0, sizeof(SSDataBlock)); - if (!pResult) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - - taosArrayPush(pResult, output); - } - - if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); - STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); - SSubmitReq *pReq = NULL; - if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), suid) != 0) { - taosArrayDestroy(pResult); - return TSDB_CODE_FAILED; - } - if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { - taosArrayDestroy(pResult); - taosMemoryFreeClear(pReq); - return TSDB_CODE_FAILED; - } - taosMemoryFreeClear(pReq); - } else { - tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); - } - - taosArrayDestroy(pResult); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - if (!pEnv) { - // only applicable when rsma env exists - return TSDB_CODE_SUCCESS; - } - - ASSERT(uid != 0); // TODO: remove later - - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaInfo *pRSmaInfo = NULL; - - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), suid); - return TSDB_CODE_SUCCESS; - } - if (!pRSmaInfo->taskInfo[0]) { - tsdbDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, REPO_ID(pTsdb), suid); - return TSDB_CODE_SUCCESS; - } - - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - // TODO: use the proper schema instead of 1, and cache STSchema in cache - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1); - if (!pTSchema) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return TSDB_CODE_FAILED; - } - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); - taosMemoryFree(pTSchema); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { - SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); - if (!pEnv) { - // only applicable when rsma env exists - return TSDB_CODE_SUCCESS; - } - - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - STbUidStore uidStore = {0}; - tsdbFetchSubmitReqSuids(pMsg, &uidStore); - - if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); - - void *pIter = taosHashIterate(uidStore.uidHash, NULL); - while (pIter) { - tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); - pIter = taosHashIterate(uidStore.uidHash, pIter); - } - - tsdbUidStoreDestory(&uidStore); - } - } - return TSDB_CODE_SUCCESS; -} - -#if 0 -/** - * @brief Get the start TS key of the last data block of one interval/sliding. - * - * @param pTsdb - * @param param - * @param result - * @return int32_t - * 1) Return 0 and fill the result if the check procedure is normal; - * 2) Return -1 if error occurs during the check procedure. - */ -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, void *smaIndex, void *result) { - const char *procedure = ""; - if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { - return -1; - } - // fill the result - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Remove the tSma data files related to param between pWin. - * - * @param pTsdb - * @param param - * @param pWin - * @return int32_t - */ -int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { - // for ("tSmaFiles of param-interval-sliding between pWin") { - // // remove the tSmaFile - // } - return TSDB_CODE_SUCCESS; -} -#endif - -// TODO: Who is responsible for resource allocate and release? -int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertTSmaDataImpl(pTsdb, indexUid, msg)) < 0) { - tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, pMsg, version)) < 0) { - tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, querySKey, nMaxResult)) < 0) { - tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { - tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c deleted file mode 100644 index a553f32bee..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define ALLOW_FORBID_FUNC - -#include "tsdb.h" - -int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) { - int ret = 0; - - if (path == NULL) return -1; - - ret = tdbOpen(path, 4096, 256, ppEnv); // use as param - - if (ret != 0) { - tsdbError("Failed to create tsdb db env, ret = %d", ret); - return -1; - } - - return 0; -} - -int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } - -static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { - const SSmaKey *pKey1 = (const SSmaKey *)arg1; - const SSmaKey *pKey2 = (const SSmaKey *)arg2; - - ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); - - if (pKey1->skey < pKey2->skey) { - return -1; - } else if (pKey1->skey > pKey2->skey) { - return 1; - } - if (pKey1->groupId < pKey2->groupId) { - return -1; - } else if (pKey1->groupId > pKey2->groupId) { - return 1; - } - - return 0; -} - -static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { - int ret; - tdb_cmpr_fn_t compFunc; - - // Create a database - compFunc = tsdbSmaKeyCmpr; - ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); - - return 0; -} - -static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } - -int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) { - // TEnv is shared by a group of SDBFile - if (!pEnv || !pDBF) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - // Open DBF - if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - tsdbCloseDBDb(pDBF->pDB); - return -1; - } - - return 0; -} - -int32_t tsdbCloseDBF(SDBFile *pDBF) { - int32_t ret = 0; - if (pDBF->pDB) { - ret = tsdbCloseDBDb(pDBF->pDB); - pDBF->pDB = NULL; - } - taosMemoryFreeClear(pDBF->path); - return ret; -} - -int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { - int32_t ret; - - ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); - if (ret < 0) { - tsdbError("Failed to create insert sma data into db, ret = %d", ret); - return -1; - } - - return 0; -} - -void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { - void *pVal = NULL; - int ret; - - ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); - - if (ret < 0) { - tsdbError("Failed to get sma data from db, ret = %d", ret); - return NULL; - } - - ASSERT(*valLen >= 0); - - // TODO: lock? - // TODO: Would the key/value be destoryed during return the data? - // TODO: How about the key is updated while value length is changed? The original value buffer would be freed - // automatically? - - return pVal; -} \ No newline at end of file From 0c31c89b38f52a61476ef51b98bad5ce59f03827 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 02:48:45 +0000 Subject: [PATCH 02/25] refact more --- include/common/tdataformat.h | 21 +++++++ source/common/src/tdataformat.c | 103 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 957b93d359..986ef5edb9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSchema SSchema; typedef struct STColumn STColumn; typedef struct STSchema STSchema; +typedef struct SValue SValue; typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; @@ -116,6 +117,26 @@ struct STSRowBuilder { STSRow2 row; }; +struct SValue { + union { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + TSKEY ts; + float f; + double d; + struct { + uint32_t nData; + uint8_t *pData; + }; + }; +}; + typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; struct SColVal { EColValT type; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34e78c21db..f762c48f44 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,6 +43,109 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); +// SValue +static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { + int32_t n = 0; + + if (IS_VAR_DATA_TYPE(type)) { + n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); + } else { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0); + break; + case TSDB_DATA_TYPE_TINYINT: + n += tPutI8(p ? p + n : p, pValue->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tPutI16(p ? p + n : p, pValue->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tPutI32(p ? p + n : p, pValue->i32); + break; + case TSDB_DATA_TYPE_BIGINT: + n += tPutI64(p ? p + n : p, pValue->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tPutFloat(p ? p + n : p, pValue->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tPutDouble(p ? p + n : p, pValue->d); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + n += tPutI64(p ? p + n : p, pValue->ts); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tPutU8(p ? p + n : p, pValue->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tPutU16(p ? p + n : p, pValue->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tPutU32(p ? p + n : p, pValue->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tPutU64(p ? p + n : p, pValue->u64); + break; + default: + ASSERT(0); + } + } + + return n; +} + +static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { + int32_t n = 0; + + if (IS_VAR_DATA_TYPE(type)) { + n += tGetBinary(p, pValue ? &pValue->pData : NULL, pValue ? &pValue->nData : NULL); + } else { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + n += tGetI8(p, &pValue->i8); + break; + case TSDB_DATA_TYPE_TINYINT: + n += tGetI8(p, &pValue->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tGetI16(p, &pValue->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tGetI32(p, &pValue->i32); + break; + case TSDB_DATA_TYPE_BIGINT: + n += tGetI64(p, &pValue->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tGetFloat(p, &pValue->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tGetDouble(p, &pValue->d); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + n += tGetI64(p, &pValue->ts); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tGetU8(p, &pValue->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tGetU16(p, &pValue->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tGetU32(p, &pValue->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tGetU64(p, &pValue->u64); + break; + default: + ASSERT(0); + } + } + + return n; +} + // STSRow2 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; From 010f1d2e2f7a080feb995aac67e8a5c54b829c3a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 02:59:55 +0000 Subject: [PATCH 03/25] more refact --- source/common/src/tdataformat.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f762c48f44..c57de4e2fc 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -103,40 +103,40 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { } else { switch (type) { case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, &pValue->i8); + n += tGetI8(p, pValue ? &pValue->i8 : NULL); break; case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, &pValue->i8); + n += tGetI8(p, pValue ? &pValue->i8 : NULL); break; case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pValue->i16); + n += tGetI16(p, pValue ? &pValue->i16 : NULL); break; case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pValue->i32); + n += tGetI32(p, pValue ? &pValue->i32 : NULL); break; case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pValue->i64); + n += tGetI64(p, pValue ? &pValue->i64 : NULL); break; case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pValue->f); + n += tGetFloat(p, pValue ? &pValue->f : NULL); break; case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pValue->d); + n += tGetDouble(p, pValue ? &pValue->d : NULL); break; case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, &pValue->ts); + n += tGetI64(p, pValue ? &pValue->ts : NULL); break; case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pValue->u8); + n += tGetU8(p, pValue ? &pValue->u8 : NULL); break; case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pValue->u16); + n += tGetU16(p, pValue ? &pValue->u16 : NULL); break; case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pValue->u32); + n += tGetU32(p, pValue ? &pValue->u32 : NULL); break; case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pValue->u64); + n += tGetU64(p, pValue ? &pValue->u64 : NULL); break; default: ASSERT(0); From d6f02761a02e0a8c430919500104c313c8bbefec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:08:29 +0000 Subject: [PATCH 04/25] more refact --- include/common/tdataformat.h | 19 ++++++++++++------- source/common/src/tdataformat.c | 12 +++++++++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 986ef5edb9..ed3a911c13 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -42,11 +42,13 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SColVal -#define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) -#define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) -#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) +// #define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) +// #define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) +// #define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) // STSRow2 +int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); @@ -54,11 +56,13 @@ void tTSRowFree(STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder +#if 0 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema); void tTSRowBuilderClear(STSRowBuilder *pBuilder); void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); +#endif // STagVal static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, @@ -137,11 +141,12 @@ struct SValue { }; }; -typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; struct SColVal { - EColValT type; - uint32_t nData; - uint8_t *pData; + int8_t isNone; + int8_t isNull; + int8_t type; + int16_t cid; + SValue value; }; struct STagVal { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c57de4e2fc..653381925c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -146,7 +146,13 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -// STSRow2 +// STSRow2 ======================================================================== +int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { + int32_t code = 0; + // TODO + return code; +} + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; @@ -215,6 +221,7 @@ void tTSRowFree(STSRow2 *pRow) { } int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { +#if 0 uint32_t n; uint8_t *p; uint8_t v; @@ -310,6 +317,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal } } +#endif return 0; } @@ -354,6 +362,7 @@ void tTSchemaDestroy(STSchema *pTSchema) { } // STSRowBuilder +#if 0 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; @@ -612,6 +621,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { return 0; } +#endif static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { From 2baee09b5863d399974c90d9423238479aac8c6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:34:38 +0000 Subject: [PATCH 05/25] more --- include/common/tdataformat.h | 1 - source/common/src/tdataformat.c | 47 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ed3a911c13..c75c5b41f2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -144,7 +144,6 @@ struct SValue { struct SColVal { int8_t isNone; int8_t isNull; - int8_t type; int16_t cid; SValue value; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 653381925c..cea633a8d9 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -44,7 +44,7 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); // SValue -static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { +static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -95,7 +95,7 @@ static int32_t tPutSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { +static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -146,6 +146,49 @@ static int32_t tGetSValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } +// SColVal +static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { + int32_t n = 0; + + ASSERT(pColVal->isNone == 0); + if (isTuple) { + ASSERT(pColVal->isNull == 0); + n += tPutValue(p ? p + n : p, &pColVal->value, type); + } else { + if (pColVal->isNull) { + // -cid means NULL + n += tPutI16v(p ? p + n : p, -pColVal->cid); + } else { + n += tPutI16v(p ? p + n : p, pColVal->cid); + n += tPutValue(p ? p + n : p, &pColVal->value, type); + } + } + + return n; +} + +static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { + int32_t n = 0; + int16_t cid; + + if (isTuple) { + n += tGetValue(p + n, pColVal ? &pColVal->value : NULL, type); + } else { + n += tGetI16v(p + n, &cid); + if (cid < 0) { + if (pColVal) { + pColVal->isNull = 1; + pColVal->cid = -cid; + } + } else { + if (pColVal) pColVal->cid = cid; + n += tGetValue(p ? p + n : p, pColVal ? &pColVal->value : NULL, type); + } + } + + return n; +} + // STSRow2 ======================================================================== int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; From 607ae9e6a4c00271b675b5cc841520956c364bf3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 03:44:44 +0000 Subject: [PATCH 06/25] refact more --- source/common/src/tdataformat.c | 43 +++++++++++++++------------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cea633a8d9..2994411693 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -99,44 +99,44 @@ static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { - n += tGetBinary(p, pValue ? &pValue->pData : NULL, pValue ? &pValue->nData : NULL); + n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); } else { switch (type) { case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, pValue ? &pValue->i8 : NULL); + n += tGetI8(p, &pValue->i8); break; case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, pValue ? &pValue->i8 : NULL); + n += tGetI8(p, &pValue->i8); break; case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, pValue ? &pValue->i16 : NULL); + n += tGetI16(p, &pValue->i16); break; case TSDB_DATA_TYPE_INT: - n += tGetI32(p, pValue ? &pValue->i32 : NULL); + n += tGetI32(p, &pValue->i32); break; case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, pValue ? &pValue->i64 : NULL); + n += tGetI64(p, &pValue->i64); break; case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, pValue ? &pValue->f : NULL); + n += tGetFloat(p, &pValue->f); break; case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, pValue ? &pValue->d : NULL); + n += tGetDouble(p, &pValue->d); break; case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, pValue ? &pValue->ts : NULL); + n += tGetI64(p, &pValue->ts); break; case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, pValue ? &pValue->u8 : NULL); + n += tGetU8(p, &pValue->u8); break; case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, pValue ? &pValue->u16 : NULL); + n += tGetU16(p, &pValue->u16); break; case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, pValue ? &pValue->u32 : NULL); + n += tGetU32(p, &pValue->u32); break; case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, pValue ? &pValue->u64 : NULL); + n += tGetU64(p, &pValue->u64); break; default: ASSERT(0); @@ -169,20 +169,17 @@ static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { int32_t n = 0; - int16_t cid; + memset(pColVal, 0, sizeof(*pColVal)); if (isTuple) { - n += tGetValue(p + n, pColVal ? &pColVal->value : NULL, type); + n += tGetValue(p + n, &pColVal->value, type); } else { - n += tGetI16v(p + n, &cid); - if (cid < 0) { - if (pColVal) { - pColVal->isNull = 1; - pColVal->cid = -cid; - } + n += tGetI16v(p + n, &pColVal->cid); + if (pColVal->cid < 0) { + pColVal->isNull = 1; + pColVal->cid = -pColVal->cid; } else { - if (pColVal) pColVal->cid = cid; - n += tGetValue(p ? p + n : p, pColVal ? &pColVal->value : NULL, type); + n += tGetValue(p + n, &pColVal->value, type); } } From 0e401a346f24420da72d961f7c20f59fe4a2b69e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 05:37:10 +0000 Subject: [PATCH 07/25] more --- include/common/tmsg.h | 16 ++++++++++++++++ include/common/tmsgdef.h | 2 ++ source/dnode/vnode/src/inc/tsdb.h | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 37e4cc5e8c..54676a0382 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2632,6 +2632,22 @@ typedef struct { int32_t tEncodeSVSubmitReq(SEncoder* pCoder, const SVSubmitReq* pReq); int32_t tDecodeSVSubmitReq(SDecoder* pCoder, SVSubmitReq* pReq); +// TDMT_VND_DELETE +typedef struct { + TSKEY sKey; + TSKEY eKey; + + // super table + char* stbName; + + // child/normal + char* tbName; +} SVDeleteReq; + +typedef struct { + // TODO +} SVDeleteRsp; + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..62517a9a14 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -226,6 +226,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "vnode-delete-data", SVDeleteReq, SVDeleteRsp) + // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2e4ff6a4ab..e5f163a03b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -33,6 +33,7 @@ extern "C" { // clang-format on // tsdbMemTable ================ +typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; typedef struct STsdbMemTable STsdbMemTable; typedef struct SMergeInfo SMergeInfo; @@ -845,6 +846,11 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } +struct STsdbRow { + int64_t version; + STSRow row; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 9b9a431b50..99a4d78c6f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -321,7 +321,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } - if(pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name); + if (pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name); if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schemaRow.version; From d68c15c5fb4b7d07ae3e21c4496bed894efa0ffd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 05:44:33 +0000 Subject: [PATCH 08/25] make it compile --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 025b2ab580..cb9bf00ed7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,6 +15,7 @@ #include "tsdb.h" +#if 0 typedef struct SMemTable SMemTable; typedef struct SMemData SMemData; typedef struct SMemSkipList SMemSkipList; @@ -377,4 +378,5 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} \ No newline at end of file +} +#endif \ No newline at end of file From e86db5fa79786aa794c679ef8ec1e97dbd31d78a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 06:57:34 +0000 Subject: [PATCH 09/25] more --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/tsdb/tsdbDelete.c | 14 +++++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 63 --------------------- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 6 +- 4 files changed, 17 insertions(+), 68 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbDelete.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b80adb9a95..dd51e2c911 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -41,7 +41,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbMemTable2.c" + # "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbDelete.c b/source/dnode/vnode/src/tsdb/tsdbDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 99a4d78c6f..7b3882aa75 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -142,69 +142,6 @@ int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter } else { fKey = tdGetKey(filterKeys[filterIter]); } -#if 0 - } else if (fKey > rowKey) { - if (isRowDel) { - pMergeInfo->rowsDeleteFailed++; - } else { - if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - - pMergeInfo->rowsInserted++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = TD_ROW_IS_DELETED(row); - } - } else { - if (isRowDel) { - ASSERT(!keepDup); - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsDeleteSucceed++; - pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - if (keepDup) { - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); - } - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = TD_ROW_IS_DELETED(row); - } - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } -#endif #if 1 } else if (fKey > rowKey) { if (isRowDel) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index cb9bf00ed7..690e1baf73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,7 +15,6 @@ #include "tsdb.h" -#if 0 typedef struct SMemTable SMemTable; typedef struct SMemData SMemData; typedef struct SMemSkipList SMemSkipList; @@ -100,7 +99,7 @@ static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); -// SMemTable +// SMemTable ======================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -378,5 +377,4 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} -#endif \ No newline at end of file +} \ No newline at end of file From 3eefc938302af19e06fe24cead3ae6ec77967120 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 07:34:39 +0000 Subject: [PATCH 10/25] more code --- source/dnode/vnode/src/inc/tsdb.h | 10 ++++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 65 ++++++++++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e5f163a03b..2283f919bc 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,11 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +typedef struct TSDBKEY TSDBKEY; + +// tsdbMemTable2.c ============================================================================================== +typedef struct SMemTable SMemTable; + // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; @@ -851,6 +856,11 @@ struct STsdbRow { STSRow row; }; +struct TSDBKEY { + int64_t version; + TSKEY ts; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 690e1baf73..f8304be783 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,27 +15,45 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; -typedef struct SMemSkipListCurosr SMemSkipListCurosr; +typedef struct SMemData SMemData; +// typedef struct SMemSkipList SMemSkipList; +// typedef struct SMemSkipListNode SMemSkipListNode; +// typedef struct SMemSkipListCurosr SMemSkipListCurosr; -#define SL_MAX_LEVEL 5 +struct SMemData { + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SMemData *pHashNext; +}; struct SMemTable { - STsdb *pTsdb; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - int32_t nHash; - int32_t nBucket; - SMemData **pBuckets; - SMemSkipListCurosr *pSlc; + STsdb *pTsdb; + int32_t nRef; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SArray *pArray; }; +int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbMemTableDestroy(SMemTable *pMemTable) { + int32_t code = 0; + // TODO + return code; +} + +#if 0 + +#define SL_MAX_LEVEL 5 + struct SMemSkipListNode { int8_t level; SMemSkipListNode *forwards[1]; // Windows does not allow 0 @@ -49,18 +67,6 @@ struct SMemSkipList { SMemSkipListNode pHead[1]; // Windows does not allow 0 }; -struct SMemData { - SMemData *pHashNext; - tb_uid_t suid; - tb_uid_t uid; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - SMemSkipList sl; -}; - struct SMemSkipListCurosr { SMemSkipList *pSl; SMemSkipListNode *pNodes[SL_MAX_LEVEL]; @@ -377,4 +383,5 @@ static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipLis } return pNode; -} \ No newline at end of file +} +#endif \ No newline at end of file From 31224d1263c666980fcf055891d6f4680f4f4256 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 07:36:21 +0000 Subject: [PATCH 11/25] more --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2283f919bc..19c05bd9cb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -45,7 +45,7 @@ typedef struct SMergeInfo SMergeInfo; typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); -void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); +void tsdbMemTableDestroy(STsdbMemTable *pMemTable); int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 88d8ee9f92..0a85cb4638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -238,7 +238,7 @@ static void tsdbStartCommit(STsdb *pRepo) { static void tsdbEndCommit(STsdb *pTsdb, int eno) { tsdbEndFSTxn(pTsdb); - tsdbMemTableDestroy(pTsdb, pTsdb->imem); + tsdbMemTableDestroy(pTsdb->imem); pTsdb->imem = NULL; tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 7b3882aa75..350e723541 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -60,7 +60,7 @@ int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { return 0; } -void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) { +void tsdbMemTableDestroy(STsdbMemTable *pMemTable) { if (pMemTable) { taosHashCleanup(pMemTable->pHashIdx); SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx); From e0b4319bd3596820969d058e924ead941e0bfd96 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 08:00:49 +0000 Subject: [PATCH 12/25] more --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/inc/tsdb.h | 11 +++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 90 ++++++++------------- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dd51e2c911..b80adb9a95 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -41,7 +41,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" - # "src/tsdb/tsdbMemTable2.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 19c05bd9cb..81dcea9917 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -33,10 +33,14 @@ extern "C" { // clang-format on typedef struct TSDBKEY TSDBKEY; +typedef struct SDelOp SDelOp; // tsdbMemTable2.c ============================================================================================== typedef struct SMemTable SMemTable; +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable); +void tsdbMemTableDestroy2(SMemTable *pMemTable); + // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; @@ -861,6 +865,13 @@ struct TSDBKEY { TSKEY ts; }; +struct SDelOp { + int64_t version; + TSKEY sKey; // included + TSKEY eKey; // included + SDelOp *pNext; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index f8304be783..ef130d1af6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -21,12 +21,12 @@ typedef struct SMemData SMemData; // typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SDelOp *pDelList; }; struct SMemTable { @@ -38,16 +38,39 @@ struct SMemTable { SArray *pArray; }; -int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { - int32_t code = 0; - // TODO +// SMemTable ============================================== +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { + int32_t code = 0; + SMemTable *pMemTable = NULL; + + pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); + if (pMemTable == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pMemTable->pTsdb = pTsdb; + pMemTable->nRef = 1; + pMemTable->minKey = (TSDBKEY){.version = -1, .ts = TSKEY_MAX}; + pMemTable->maxKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MIN}; + pMemTable->nRows = 0; + pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); + if (pMemTable->pArray == NULL) { + taosMemoryFree(pMemTable); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + *ppMemTable = pMemTable; + return code; + +_err: + *ppMemTable = NULL; return code; } -int32_t tsdbMemTableDestroy(SMemTable *pMemTable) { - int32_t code = 0; +void tsdbMemTableDestroy2(SMemTable *pMemTable) { + ASSERT(0); // TODO - return code; } #if 0 @@ -106,49 +129,6 @@ static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); // SMemTable ======================== -int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { - SMemTable *pMemTb = NULL; - - pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); - if (pMemTb == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMemTb->pTsdb = pTsdb; - pMemTb->minKey = TSKEY_MAX; - pMemTb->maxKey = TSKEY_MIN; - pMemTb->minVer = -1; - pMemTb->maxVer = -1; - pMemTb->nRows = 0; - pMemTb->nHash = 0; - pMemTb->nBucket = 1024; - pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); - if (pMemTb->pBuckets == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pMemTb); - return -1; - } - if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pMemTb->pBuckets); - taosMemoryFree(pMemTb); - } - - *ppMemTb = pMemTb; - return 0; -} - -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { - if (pMemTb) { - // loop to destroy the contents (todo) - tsdbMemSkipListCursorDestroy(pMemTb->pSlc); - taosMemoryFree(pMemTb->pBuckets); - taosMemoryFree(pMemTb); - } - return 0; -} - int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { SMemData *pMemData; STsdb *pTsdb = pMemTb->pTsdb; From 81ef708c2ada4cc9a34193884c1ed0f3b8e77f94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 08:42:46 +0000 Subject: [PATCH 13/25] more --- source/dnode/vnode/src/inc/tsdb.h | 8 +-- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 62 +++++++++++++++++++-- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 81dcea9917..de69096902 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,7 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; typedef struct SDelOp SDelOp; @@ -855,9 +856,9 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } -struct STsdbRow { - int64_t version; - STSRow row; +struct TSDBROW { + int64_t version; + STSRow2 *pRow; }; struct TSDBKEY { @@ -869,7 +870,6 @@ struct SDelOp { int64_t version; TSKEY sKey; // included TSKEY eKey; // included - SDelOp *pNext; }; #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ef130d1af6..4494dafc44 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -26,7 +26,7 @@ struct SMemData { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SDelOp *pDelList; + SArray *aDelOp; // SArray }; struct SMemTable { @@ -35,9 +35,11 @@ struct SMemTable { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SArray *pArray; + SArray *pArray; // SArray }; +static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); + // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -69,11 +71,61 @@ _err: } void tsdbMemTableDestroy2(SMemTable *pMemTable) { - ASSERT(0); - // TODO + taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/); + taosMemoryFree(pMemTable); } -#if 0 +int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) { + int32_t code = 0; + SMemData *pMemData; + + // check if table exists + { + // TODO + } + + code = tsdbGetOrCreateTbData(pTsdb, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); + if (code) { + tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _err; + } + + // do insert + + return code; + +_err: + return code; +} + +int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { + int32_t code = 0; + SMemData *pMemData = NULL; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + + // search + pMemData = (SMemData *)taosbsearch(NULL, pMemTable->pArray->pData, taosArrayGetSize(pMemTable->pArray), + sizeof(SMemData *), NULL, TD_GE); + + if (pMemData == NULL) { + // not found, create one + } + + *ppMemData = pMemData; + return code; + +_err: + *ppMemData = NULL; + return code; +} + +#if 0 //==================================================================================== #define SL_MAX_LEVEL 5 From 670a90e55a656c2c4c49bcec690a21ce4828fafc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 10:11:32 +0000 Subject: [PATCH 14/25] feat: tsdb multi-version --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 127 +++++++++++++++++--- 2 files changed, 109 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index de69096902..fe6ecdcbb1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -870,6 +870,7 @@ struct SDelOp { int64_t version; TSKEY sKey; // included TSKEY eKey; // included + SDelOp *pNext; }; #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 4494dafc44..049364e9d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -26,7 +26,8 @@ struct SMemData { TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; - SArray *aDelOp; // SArray + SDelOp *delOpHead; + SDelOp *delOpTail; }; struct SMemTable { @@ -38,7 +39,8 @@ struct SMemTable { SArray *pArray; // SArray }; -static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); +static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); +static int memDataPCmprFn(const void *p1, const void *p2); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -52,8 +54,8 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { } pMemTable->pTsdb = pTsdb; pMemTable->nRef = 1; - pMemTable->minKey = (TSDBKEY){.version = -1, .ts = TSKEY_MAX}; - pMemTable->maxKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MIN}; + pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; + pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; pMemTable->nRows = 0; pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); if (pMemTable->pArray == NULL) { @@ -76,15 +78,17 @@ void tsdbMemTableDestroy2(SMemTable *pMemTable) { } int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) { - int32_t code = 0; - SMemData *pMemData; + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO + SMemData *pMemData; + + ASSERT(pMemTable); - // check if table exists { - // TODO + // check if table exists (todo) } - code = tsdbGetOrCreateTbData(pTsdb, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); + code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); if (code) { tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _err; @@ -99,24 +103,90 @@ _err: } int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { - int32_t code = 0; - // TODO + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO + SMemData *pMemData; + SVBufPool *pPool = pTsdb->pVnode->inUse; + + ASSERT(pMemTable); + + { + // check if table exists (todo) + } + + code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData); + if (code) { + goto _err; + } + + // do delete + SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp)); + if (pDelOp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pDelOp->version = version; + pDelOp->sKey = sKey; + pDelOp->eKey = eKey; + pDelOp->pNext = NULL; + if (pMemData->delOpHead == NULL) { + ASSERT(pMemData->delOpTail == NULL); + pMemData->delOpHead = pMemData->delOpTail = pDelOp; + } else { + pMemData->delOpTail->pNext = pDelOp; + pMemData->delOpTail = pDelOp; + } + + { + // update the state of pMemTable, pMemData, last and lastrow (todo) + } + + tsdbDebug("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + " since %s", + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); + return code; + +_err: + tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + " since %s", + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; } -static int32_t tsdbGetOrCreateTbData(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { +static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { int32_t code = 0; + int32_t idx = 0; + SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid}; SMemData *pMemData = NULL; - SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; - // search - pMemData = (SMemData *)taosbsearch(NULL, pMemTable->pArray->pData, taosArrayGetSize(pMemTable->pArray), - sizeof(SMemData *), NULL, TD_GE); - - if (pMemData == NULL) { - // not found, create one + // get + idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); + if (idx >= 0) { + pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx); + if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit; } + // create + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + if (pMemData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pMemData->suid = suid; + pMemData->uid = uid; + pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; + pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; + pMemData->nRows = 0; + pMemData->delOpHead = pMemData->delOpTail = NULL; + + if (idx < 0) idx = 0; + if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + +_exit: *ppMemData = pMemData; return code; @@ -125,6 +195,25 @@ _err: return code; } +static int memDataPCmprFn(const void *p1, const void *p2) { + SMemData *pMemData1 = *(SMemData **)p1; + SMemData *pMemData2 = *(SMemData **)p2; + + if (pMemData1->suid < pMemData2->suid) { + return -1; + } else if (pMemData1->suid > pMemData2->suid) { + return 1; + } + + if (pMemData1->uid < pMemData2->uid) { + return -1; + } else if (pMemData1->uid > pMemData2->uid) { + return 1; + } + + return 0; +} + #if 0 //==================================================================================== #define SL_MAX_LEVEL 5 From 671f613c02ad5df7bb5ad145f3c1db17a3a58071 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 10:20:58 +0000 Subject: [PATCH 15/25] feat: tsdb multi-version --- source/dnode/vnode/src/inc/tsdb.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index fe6ecdcbb1..4f22ce768f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -857,8 +857,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { } struct TSDBROW { - int64_t version; - STSRow2 *pRow; + int64_t version; + STSRow2 tsRow; }; struct TSDBKEY { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 049364e9d7..aab190b0bb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -81,6 +81,7 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit int32_t code = 0; SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO SMemData *pMemData; + TSDBROW row = {.version = version}; ASSERT(pMemTable); @@ -95,6 +96,15 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit } // do insert + uint32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + while (n < pSubmitBlk->nData) { + n += tGetTSRow(p + n, &row.tsRow); + + ASSERT(n <= pSubmitBlk->nData); + + // TODO + } return code; From 73cbc6f717cbf3815ec00fc34bedac041af90cd6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 12:53:23 +0000 Subject: [PATCH 16/25] feat: vnode multi-version --- source/dnode/vnode/src/inc/tsdb.h | 21 +++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 139 +++++++++++++------- 2 files changed, 114 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4f22ce768f..a62b4c4409 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -36,6 +36,8 @@ typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; typedef struct SDelOp SDelOp; +static int tsdbKeyCmprFn(const void *p1, const void *p2); + // tsdbMemTable2.c ============================================================================================== typedef struct SMemTable SMemTable; @@ -873,6 +875,25 @@ struct SDelOp { SDelOp *pNext; }; +static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { + TSDBKEY *pKey1 = (TSDBKEY *)p1; + TSDBKEY *pKey2 = (TSDBKEY *)p2; + + if (pKey1->ts < pKey2->ts) { + return -1; + } else if (pKey1->ts > pKey2->ts) { + return 1; + } + + if (pKey1->version < pKey2->version) { + return -1; + } else if (pKey1->version > pKey2->version) { + return 1; + } + + return 0; +} + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index aab190b0bb..cbc9fef686 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,19 +15,33 @@ #include "tsdb.h" -typedef struct SMemData SMemData; -// typedef struct SMemSkipList SMemSkipList; -// typedef struct SMemSkipListNode SMemSkipListNode; -// typedef struct SMemSkipListCurosr SMemSkipListCurosr; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; + +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[0]; +}; + +struct SMemSkipList { + uint32_t seed; + int32_t size; + int8_t maxLevel; + int8_t level; + SMemSkipListNode *pHead; + SMemSkipListNode *pTail; +}; struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SDelOp *delOpHead; - SDelOp *delOpTail; + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SDelOp *delOpHead; + SDelOp *delOpTail; + SMemSkipList sl; }; struct SMemTable { @@ -39,8 +53,17 @@ struct SMemTable { SArray *pArray; // SArray }; +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); +static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); +static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -96,16 +119,41 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit } // do insert - uint32_t n = 0; - uint8_t *p = pSubmitBlk->pData; + int32_t nt; + uint8_t *pt; + int32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + SVBufPool *pPool = pTsdb->pVnode->inUse; + int8_t level; + SMemSkipListNode *pNode; while (n < pSubmitBlk->nData) { - n += tGetTSRow(p + n, &row.tsRow); + nt = tGetTSRow(p + n, &row.tsRow); + n += nt; ASSERT(n <= pSubmitBlk->nData); - // TODO + // build the node + level = tsdbMemSkipListRandLevel(&pMemData->sl); + pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + nt + sizeof(version)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pNode->level = level; + tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), &row); + + // put the node (todo) + + pMemData->nRows++; + + // set info + if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row; + if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row; } + if (tsdbKeyCmprFn(&pMemTable->minKey, &pMemData->minKey) < 0) pMemTable->minKey = pMemData->minKey; + if (tsdbKeyCmprFn(&pMemTable->maxKey, &pMemData->maxKey) > 0) pMemTable->maxKey = pMemData->maxKey; + return code; _err: @@ -224,23 +272,40 @@ static int memDataPCmprFn(const void *p1, const void *p2) { return 0; } +static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = 0; + + n += tPutI64(p ? p + n : p, pRow->version); + n += tPutTSRow(p ? p + n : p, &pRow->tsRow); + + return n; +} + +static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = 0; + + n += tGetI64(p + n, &pRow->version); + n += tGetTSRow(p + n, &pRow->tsRow); + + return n; +} + +static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { + int8_t level = 1; + int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); + const uint32_t factor = 4; + + while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { + level++; + } + + return level; +} + #if 0 //==================================================================================== #define SL_MAX_LEVEL 5 -struct SMemSkipListNode { - int8_t level; - SMemSkipListNode *forwards[1]; // Windows does not allow 0 -}; - -struct SMemSkipList { - uint32_t seed; - int8_t maxLevel; - int8_t level; - int32_t size; - SMemSkipListNode pHead[1]; // Windows does not allow 0 -}; - struct SMemSkipListCurosr { SMemSkipList *pSl; SMemSkipListNode *pNodes[SL_MAX_LEVEL]; @@ -254,12 +319,6 @@ typedef struct { #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) -#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) -#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) -#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) - #define SL_HEAD_NODE(sl) ((sl)->pHead) #define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) #define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l) @@ -390,18 +449,6 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { - int8_t level = 1; - int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - const uint32_t factor = 4; - - while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { - level++; - } - - return level; -} - static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) { if (tEncodeI64(pEncoder, pRow->version) < 0) return -1; if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1; From 874a29d3eba2a43994f997a1be61015dad8af8c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 May 2022 13:14:30 +0000 Subject: [PATCH 17/25] feat: vnode multi-version --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index cbc9fef686..94c88a14ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,6 @@ struct SMemData { tb_uid_t uid; TSDBKEY minKey; TSDBKEY maxKey; - int64_t nRows; SDelOp *delOpHead; SDelOp *delOpTail; SMemSkipList sl; @@ -59,6 +58,9 @@ struct SMemTable { #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) #define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) +#define SL_HEAD_FORWARD(sl, l) SL_NODE_FORWARD((sl)->pHead, l) +#define SL_TAIL_BACKWARD(sl, l) SL_NODE_FORWARD((sl)->pTail, l) + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); @@ -144,8 +146,6 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit // put the node (todo) - pMemData->nRows++; - // set info if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row; if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row; @@ -217,6 +217,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid}; SMemData *pMemData = NULL; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; // get idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); @@ -226,7 +227,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui } // create - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2); if (pMemData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -235,8 +236,18 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui pMemData->uid = uid; pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; - pMemData->nRows = 0; pMemData->delOpHead = pMemData->delOpTail = NULL; + pMemData->sl.seed = taosRand(); + pMemData->sl.size = 0; + pMemData->sl.maxLevel = maxLevel; + pMemData->sl.level = 0; + pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1]; + pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_HALF_SIZE(maxLevel)); + + for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { + SL_HEAD_FORWARD(&pMemData->sl, iLevel) = pMemData->sl.pTail; + SL_TAIL_BACKWARD(&pMemData->sl, iLevel) = pMemData->sl.pHead; + } if (idx < 0) idx = 0; if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { From 646357f1dcdf3c583d820cfd13de8998c9d5914c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 03:08:46 +0000 Subject: [PATCH 18/25] more --- include/common/tdataformat.h | 24 ++--- source/common/src/tdataformat.c | 182 ++++++++++++++++++++++++++++++-- 2 files changed, 186 insertions(+), 20 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index c75c5b41f2..b142c36b35 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,11 +48,11 @@ void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); -void tTSRowFree(STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder @@ -123,17 +123,17 @@ struct STSRowBuilder { struct SValue { union { - int8_t i8; - uint8_t u8; - int16_t i16; - uint16_t u16; - int32_t i32; - uint32_t u32; - int64_t i64; - uint64_t u64; - TSKEY ts; - float f; - double d; + int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT + uint8_t u8; // TSDB_DATA_TYPE_UTINYINT + int16_t i16; // TSDB_DATA_TYPE_SMALLINT + uint16_t u16; // TSDB_DATA_TYPE_USMALLINT + int32_t i32; // TSDB_DATA_TYPE_INT + uint32_t u32; // TSDB_DATA_TYPE_UINT + int64_t i64; // TSDB_DATA_TYPE_BIGINT + uint64_t u64; // TSDB_DATA_TYPE_UBIGINT + TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP + float f; // TSDB_DATA_TYPE_FLOAT + double d; // TSDB_DATA_TYPE_DOUBLE struct { uint32_t nData; uint8_t *pData; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 2994411693..69f859bb3b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -21,15 +21,10 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); -typedef struct SKVIdx { - int32_t cid; - int32_t offset; -} SKVIdx; - #pragma pack(push, 1) typedef struct { int16_t nCols; - SKVIdx idx[]; + uint8_t idx[]; } STSKVRow; #pragma pack(pop) @@ -44,7 +39,7 @@ typedef struct { static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); // SValue -static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -95,7 +90,7 @@ static int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -static int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; if (IS_VAR_DATA_TYPE(type)) { @@ -187,6 +182,177 @@ static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu } // STSRow2 ======================================================================== +static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { + int32_t nColVal = taosArrayGetSize(pArray); + STColumn *pTColumn; + SColVal *pColVal; + + ASSERT(nColVal > 0); + + pRow->sver = pTSchema->version; + + // ts + pTColumn = &pTSchema->columns[0]; + pColVal = (SColVal *)taosArrayGet(pArray, 0); + + ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + pRow->ts = pColVal->value.ts; + + // other fields + int32_t iColVal = 1; + int32_t bidx; + uint32_t nv = 0; + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t flags = 0; + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + bidx = iColumn - 1; + pTColumn = &pTSchema->columns[iColumn]; + + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + if (pColVal->isNone) { + goto _set_none; + } else if (pColVal->isNull) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + flags |= TSROW_HAS_NONE; + // SET_BIT2(pb, bidx, 0); (todo) + continue; + + _set_null: + flags != TSROW_HAS_NULL; + // SET_BIT2(pb, bidx, 1); (todo) + continue; + + _set_value: + flags != TSROW_HAS_VAL; + // SET_BIT2(pb, bidx, 2); (todo) + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); + } else { + tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); + } + continue; + } + + ASSERT(flags); + switch (flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + pRow->nData = 0; + break; + case TSROW_HAS_VAL: + pRow->nData = pTSchema->flen + nv; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1); + break; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + pRow->nData = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; + break; + default: + break; + } +} + +static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { + int32_t nColVal = taosArrayGetSize(pArray); + STColumn *pTColumn; + SColVal *pColVal; + + ASSERT(nColVal > 0); + + pRow->sver = pTSchema->version; + + // ts + pTColumn = &pTSchema->columns[0]; + pColVal = (SColVal *)taosArrayGet(pArray, 0); + + ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + pRow->ts = pColVal->value.ts; + + // other fields + int32_t iColVal = 1; + uint32_t nv = 0; + uint8_t *pv = NULL; + uint8_t *pidx = NULL; + uint8_t flags = 0; + int16_t nCol = 0; + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + if (pColVal->isNone) { + goto _set_none; + } else if (pColVal->isNull) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + flags |= TSROW_HAS_NONE; + continue; + + _set_null: + flags != TSROW_HAS_NULL; + pidx[nCol++] = nv; + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + continue; + + _set_value: + flags != TSROW_HAS_VAL; + pidx[nCol++] = nv; + nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + continue; + } +} + int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; // TODO From be2bfd513ef43b23db05776ba6cda1e6e95c8fc0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 03:16:09 +0000 Subject: [PATCH 19/25] more --- source/common/src/tdataformat.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 69f859bb3b..81328aba1b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -351,11 +351,33 @@ static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; } + + if (nv <= UINT8_MAX) { + // small + } else if (nv <= UINT16_MAX) { + // mid + } else { + // large + } } +// try-decide-build int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; - // TODO + STSRow2 rowT = {0}; + STSRow2 rowM = {0}; + + // try + tTupleTSRowNew(pArray, pTSchema, &rowT); + tMapTSRowNew(pArray, pTSchema, &rowM); + + // decide & build + if (rowT.nData <= rowM.nData) { + tTupleTSRowNew(pArray, pTSchema, &rowT); + } else { + tMapTSRowNew(pArray, pTSchema, &rowM); + } + return code; } From 540923c6a4d19e1121a3a44270fcdf43c2cb874c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 07:10:33 +0000 Subject: [PATCH 20/25] feat: data format --- include/common/tdataformat.h | 17 +- source/common/src/tdataformat.c | 363 +++++++++++++++++++------------- 2 files changed, 218 insertions(+), 162 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b142c36b35..f72ad49416 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -41,19 +41,14 @@ typedef struct STag STag; int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); -// SColVal -// #define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) -// #define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) -// #define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) - // STSRow2 int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); void tTSRowFree(STSRow2 *pRow); - +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); -int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder #if 0 @@ -98,7 +93,9 @@ struct STSchema { #define TSROW_HAS_NONE ((uint8_t)0x1) #define TSROW_HAS_NULL ((uint8_t)0x2U) #define TSROW_HAS_VAL ((uint8_t)0x4U) -#define TSROW_KV_ROW ((uint8_t)0x10U) +#define TSROW_KV_SMALL ((uint8_t)0x10U) +#define TSROW_KV_MID ((uint8_t)0x20U) +#define TSROW_KV_BIG ((uint8_t)0x40U) struct STSRow2 { TSKEY ts; uint8_t flags; @@ -142,9 +139,9 @@ struct SValue { }; struct SColVal { + int16_t cid; int8_t isNone; int8_t isNull; - int16_t cid; SValue value; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 81328aba1b..28015f6bd4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -162,25 +162,6 @@ static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTu return n; } -static int32_t tGetColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { - int32_t n = 0; - - memset(pColVal, 0, sizeof(*pColVal)); - if (isTuple) { - n += tGetValue(p + n, &pColVal->value, type); - } else { - n += tGetI16v(p + n, &pColVal->cid); - if (pColVal->cid < 0) { - pColVal->isNull = 1; - pColVal->cid = -pColVal->cid; - } else { - n += tGetValue(p + n, &pColVal->value, type); - } - } - - return n; -} - // STSRow2 ======================================================================== static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { int32_t nColVal = taosArrayGetSize(pArray); @@ -381,6 +362,206 @@ int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { return code; } +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { + int32_t code = 0; + + (*ppRow) = (STSRow2 *)taosMemoryMalloc(sizeof(**ppRow)); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + **ppRow = *pRow; + (*ppRow)->pData = NULL; + + if (pRow->nData) { + (*ppRow)->pData = taosMemoryMalloc(pRow->nData); + if ((*ppRow)->pData == NULL) { + taosMemoryFree(*ppRow); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy((*ppRow)->pData, pRow->pData, pRow->nData); + } + +_exit: + return code; +} + +void tTSRowFree(STSRow2 *pRow) { + if (pRow) { + if (pRow->pData) taosMemoryFree(pRow->pData); + taosMemoryFree(pRow); + } +} + +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint8_t isTuple = (pRow->flags & 0xf0 == 0) ? 1 : 0; + STColumn *pTColumn = &pTSchema->columns[iCol]; + uint8_t flags = pRow->flags & (uint8_t)0xf; + SValue value; + + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(flags); + ASSERT(pRow->sver == pTSchema->version); + + if (iCol == 0) { + value.ts = pRow->ts; + goto _return_value; + } + + if (flags == TSROW_HAS_NONE) { + *pColVal = (SColVal){.isNone = 1, .cid = pTColumn->colId}; + return; + } else if (flags == TSROW_HAS_NONE) { + *pColVal = (SColVal){.isNull = 1, .cid = pTColumn->colId}; + return; + } + + ASSERT(pRow->nData && pRow->pData); + + if (isTuple) { + uint8_t *pb = pRow->pData; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t *p; + uint8_t b; + + // bit + switch (flags) { + case TSROW_HAS_VAL: + pf = pb; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + goto _return_null; + } + case TSROW_HAS_VAL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_null; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT2(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else if (b == 1) { + goto _return_null; + } else { + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + break; + } + default: + ASSERT(0); + } + + ASSERT(pf); + + p = pf + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pv = pf + pTSchema->flen; + p = pv + *(VarDataOffsetT *)p; + } + tGetValue(p, &value, pTColumn->type); + goto _return_value; + } else { + STSKVRow *pRowK = (STSKVRow *)pRow->pData; + int16_t lidx = 0; + int16_t ridx = pRowK->nCols - 1; + uint8_t *p; + int16_t midx; + uint32_t n; + int16_t cid; + + ASSERT(pRowK->nCols > 0); + + if (pRow->flags & TSROW_KV_SMALL) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_MID) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_BIG) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + } else { + ASSERT(0); + } + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; + + if (pRow->flags & TSROW_KV_SMALL) { + n = ((uint8_t *)pRowK->idx)[midx]; + } else if (pRow->flags & TSROW_KV_MID) { + n = ((uint16_t *)pRowK->idx)[midx]; + } else { + n = ((uint32_t *)pRowK->idx)[midx]; + } + + n += tGetI16v(p + n, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + goto _return_null; + } else { + n += tGetValue(p + n, &value, pTColumn->type); + goto _return_value; + } + + return; + } else if (TABS(cid) > pTColumn->colId) { + ridx = midx - 1; + } else { + lidx = midx + 1; + } + } + + // not found, return NONE + goto _return_none; + } + +_return_none: + *pColVal = (SColVal){.cid = pTColumn->colId, .isNone = 1}; + return; + +_return_null: + *pColVal = (SColVal){.cid = pTColumn->colId, .isNull = 1}; + return; + +_return_value: + *pColVal = (SColVal){.cid = pTColumn->colId, .value = value}; + return; +} + +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { + int32_t code = 0; + SColVal cv; + + (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (*ppArray == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + tTSRowGet(pRow, pTSchema, iColumn, &cv); + taosArrayPush(*ppArray, &cv); + } + +_exit: + return code; +} + int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; @@ -393,8 +574,11 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: + ASSERT(pRow->nData == 0); + ASSERT(pRow->pData == NULL); break; default: + ASSERT(pRow->nData && pRow->pData); n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData); break; } @@ -404,151 +588,26 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; - uint8_t flags; - n += tGetI64(p + n, pRow ? &pRow->ts : NULL); - n += tGetI8(p + n, pRow ? &pRow->flags : &flags); - n += tGetI32v(p + n, pRow ? &pRow->sver : NULL); + n += tGetI64(p + n, &pRow->ts); + n += tGetI8(p + n, &pRow->flags); + n += tGetI32v(p + n, &pRow->sver); - if (pRow) flags = pRow->flags; - switch (flags & 0xf) { + ASSERT(pRow->flags); + switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: + pRow->nData = 0; + pRow->pData = NULL; break; default: - n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL); + n += tGetBinary(p + n, &pRow->pData, &pRow->nData); break; } return n; } -int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) { - (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData); - if (*ppRow == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - (*ppRow)->ts = pRow->ts; - (*ppRow)->flags = pRow->flags; - (*ppRow)->sver = pRow->sver; - (*ppRow)->nData = pRow->nData; - if (pRow->nData) { - (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]); - memcpy((*ppRow)->pData, pRow->pData, pRow->nData); - } else { - (*ppRow)->pData = NULL; - } - - return 0; -} - -void tTSRowFree(STSRow2 *pRow) { - if (pRow) taosMemoryFree(pRow); -} - -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { -#if 0 - uint32_t n; - uint8_t *p; - uint8_t v; - int32_t bidx = iCol - 1; - STColumn *pTColumn = &pTSchema->columns[iCol]; - STSKVRow *pTSKVRow; - SKVIdx *pKVIdx; - - ASSERT(iCol != 0); - ASSERT(pTColumn->colId != 0); - - ASSERT(pRow->flags & 0xf != 0); - switch (pRow->flags & 0xf) { - case TSROW_HAS_NONE: - *pColVal = ColValNONE; - return 0; - case TSROW_HAS_NULL: - *pColVal = ColValNULL; - return 0; - } - - if (TSROW_IS_KV_ROW(pRow)) { - ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL); - - pTSKVRow = (STSKVRow *)pRow->pData; - pKVIdx = - bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); - if (pKVIdx == NULL) { - *pColVal = ColValNONE; - } else if (pKVIdx->offset < 0) { - *pColVal = ColValNULL; - } else { - p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; - pColVal->type = COL_VAL_DATA; - tGetBinary(p, &pColVal->pData, &pColVal->nData); - } - } else { - // get bitmap - p = pRow->pData; - switch (pRow->flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - v = GET_BIT1(p, bidx); - if (v == 0) { - *pColVal = ColValNONE; - } else { - *pColVal = ColValNULL; - } - return 0; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - v = GET_BIT1(p, bidx); - if (v == 1) { - p = p + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } else { - *pColVal = ColValNONE; - return 0; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = GET_BIT1(p, bidx); - if (v == 1) { - p = p + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } else { - *pColVal = ColValNULL; - return 0; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - v = GET_BIT2(p, bidx); - if (v == 0) { - *pColVal = ColValNONE; - return 0; - } else if (v == 1) { - *pColVal = ColValNULL; - return 0; - } else if (v == 2) { - p = p + BIT2_SIZE(pTSchema->numOfCols - 1); - break; - } else { - ASSERT(0); - } - default: - break; - } - - // get real value - p = p + pTColumn->offset; - pColVal->type = COL_VAL_DATA; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData); - } else { - pColVal->pData = p; - pColVal->nData = pTColumn->bytes; - } - } - -#endif - return 0; -} - // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); From 62e01aba2870d14707a0393da40d18ad2c970e2b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 07:16:18 +0000 Subject: [PATCH 21/25] refact: more --- source/common/src/tdataformat.c | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 28015f6bd4..a8c263dd7d 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -141,27 +141,6 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } -// SColVal -static int32_t tPutColVal(uint8_t *p, SColVal *pColVal, int8_t type, int8_t isTuple) { - int32_t n = 0; - - ASSERT(pColVal->isNone == 0); - if (isTuple) { - ASSERT(pColVal->isNull == 0); - n += tPutValue(p ? p + n : p, &pColVal->value, type); - } else { - if (pColVal->isNull) { - // -cid means NULL - n += tPutI16v(p ? p + n : p, -pColVal->cid); - } else { - n += tPutI16v(p ? p + n : p, pColVal->cid); - n += tPutValue(p ? p + n : p, &pColVal->value, type); - } - } - - return n; -} - // STSRow2 ======================================================================== static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { int32_t nColVal = taosArrayGetSize(pArray); @@ -232,9 +211,9 @@ static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { flags != TSROW_HAS_VAL; // SET_BIT2(pb, bidx, 2); (todo) if (IS_VAR_DATA_TYPE(pTColumn->type)) { - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); } else { - tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); + // tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); } continue; } @@ -323,13 +302,13 @@ static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { _set_null: flags != TSROW_HAS_NULL; pidx[nCol++] = nv; - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; _set_value: flags != TSROW_HAS_VAL; pidx[nCol++] = nv; - nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); + // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); continue; } From c2d95cbc0d1e47dbdaceedeecbbc29bad6c085aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 07:28:57 +0000 Subject: [PATCH 22/25] more --- include/common/tdataformat.h | 5 ++++- source/common/src/tdataformat.c | 12 +++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f72ad49416..cc311228b7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -42,7 +42,10 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 -int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +#define COL_VAL_NONE(CID) ((SColVal){.cid = (CID), .isNone = 1}) +#define COL_VAL_NULL(CID) ((SColVal){.cid = (CID), .isNull = 1}) +#define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .value = (V)}) + int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); void tTSRowFree(STSRow2 *pRow); void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a8c263dd7d..4dccdcb17e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -389,11 +389,9 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal } if (flags == TSROW_HAS_NONE) { - *pColVal = (SColVal){.isNone = 1, .cid = pTColumn->colId}; - return; + goto _return_none; } else if (flags == TSROW_HAS_NONE) { - *pColVal = (SColVal){.isNull = 1, .cid = pTColumn->colId}; - return; + goto _return_null; } ASSERT(pRow->nData && pRow->pData); @@ -510,15 +508,15 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal } _return_none: - *pColVal = (SColVal){.cid = pTColumn->colId, .isNone = 1}; + *pColVal = COL_VAL_NONE(pTColumn->colId); return; _return_null: - *pColVal = (SColVal){.cid = pTColumn->colId, .isNull = 1}; + *pColVal = COL_VAL_NULL(pTColumn->colId); return; _return_value: - *pColVal = (SColVal){.cid = pTColumn->colId, .value = value}; + *pColVal = COL_VAL_VALUE(pTColumn->colId, value); return; } From 7e421c0092eefd04e00f54e13572ea0abb502b61 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:04:06 +0000 Subject: [PATCH 23/25] make it compile --- include/util/tencode.h | 187 ++++++++++++++++++++++++++++++++++------- 1 file changed, 157 insertions(+), 30 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index bde2185b02..3e1d675870 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -461,64 +461,151 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { } // =========================================== -#define tPutV(p, v) \ - int32_t n = 0; \ - for (;;) { \ - if (v <= 0x7f) { \ - if (p) p[n] = v; \ - n++; \ - break; \ - } \ - if (p) p[n] = (v & 0x7f) | 0x80; \ - n++; \ - v >>= 7; \ - } \ - return n; +#define tPutV(p, v) \ + do { \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; \ + } while (0) -#define tGetV(p, v) \ - int32_t n = 0; \ - if (v) *v = 0; \ - for (;;) { \ - if (p[n] <= 0x7f) { \ - if (v) (*v) |= (p[n] << (7 * n)); \ - n++; \ - break; \ - } \ - if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ - n++; \ - } \ - return n; +#define tGetV(p, v) \ + do { \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; \ + } while (0) // PUT +static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { + if (p) ((uint8_t*)p)[0] = v; + return sizeof(uint8_t); +} + static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { if (p) ((int8_t*)p)[0] = v; return sizeof(int8_t); } +static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { + if (p) ((uint16_t*)p)[0] = v; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { + if (p) ((int16_t*)p)[0] = v; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { + if (p) ((uint32_t*)p)[0] = v; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { + if (p) ((int32_t*)p)[0] = v; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { + if (p) ((uint64_t*)p)[0] = v; + return sizeof(uint64_t); +} + static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { if (p) ((int64_t*)p)[0] = v; return sizeof(int64_t); } -static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v) } +static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { + union { + uint32_t ui; + float f; + } v = {.f = f}; + + return tPutU32(p, v.ui); +} + +static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { + union { + uint64_t ui; + double d; + } v = {.d = d}; + + return tPutU64(p, v.ui); +} + +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } -static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v) } +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } +static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } + +// GET +static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { + if (v) *v = ((uint8_t*)p)[0]; + return sizeof(uint8_t); +} + static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { if (v) *v = ((int8_t*)p)[0]; return sizeof(int8_t); } +static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { + if (v) *v = ((uint16_t*)p)[0]; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { + if (v) *v = ((int16_t*)p)[0]; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { + if (v) *v = ((uint32_t*)p)[0]; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { + if (v) *v = ((int32_t*)p)[0]; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { + if (v) *v = ((uint64_t*)p)[0]; + return sizeof(uint64_t); +} + static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { if (v) *v = ((int64_t*)p)[0]; return sizeof(int64_t); } -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v) } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { int32_t n; @@ -530,7 +617,7 @@ static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { return n; } -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v) } +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { int32_t n; @@ -542,6 +629,46 @@ static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { return n; } +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { + int32_t n; + uint64_t tv; + + n = tGetU64v(p, &tv); + if (v) *v = ZIGZAGD(int64_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { + int32_t n = 0; + + union { + uint32_t ui; + float f; + } v; + + n = tGetU32(p, &v.ui); + + *f = v.f; + return n; +} + +static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { + int32_t n = 0; + + union { + uint64_t ui; + double d; + } v; + + n = tGetU64(p, &v.ui); + + *d = v.d; + return n; +} + // ===================== static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { int n = 0; From 8c53f98184cd7a686a43bd03c888068ef9b3a1c1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:31:54 +0000 Subject: [PATCH 24/25] make it pass windows comile --- include/common/tmsg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f38a0f8551..83ea44cd86 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -945,7 +945,6 @@ typedef struct { int64_t timeInFetchQueue; } SQnodeLoad; - typedef struct { int32_t sver; // software version int64_t dnodeVer; // dnode table version in sdb @@ -1977,7 +1976,7 @@ typedef struct { int8_t killConnection; int8_t align[3]; SEpSet epSet; - SArray *pQnodeList; + SArray* pQnodeList; } SQueryHbRspBasic; typedef struct { @@ -2676,6 +2675,7 @@ typedef struct { } SVDeleteReq; typedef struct { + int32_t code; // TODO } SVDeleteRsp; From 6f94998e062475950dcb851be37f622f8b42d8b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jun 2022 08:34:03 +0000 Subject: [PATCH 25/25] make it pass windows compile --- include/util/tencode.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 3e1d675870..a13afd4448 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -538,7 +538,8 @@ static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { union { uint32_t ui; float f; - } v = {.f = f}; + } v; + v.f = f; return tPutU32(p, v.ui); } @@ -547,7 +548,8 @@ static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { union { uint64_t ui; double d; - } v = {.d = d}; + } v; + v.d = d; return tPutU64(p, v.ui); }