diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/Kconfig b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/Kconfig new file mode 100644 index 000000000..d36030958 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/Kconfig @@ -0,0 +1,36 @@ +menu "TableStorage" + + config TABLE_STORAGE + bool "Using Table storage system" + default n + +if TABLE_STORAGE + + config TABLE_STORAGE_BASIC + bool "Enable TableStorage Basic" + default y + + config TABLE_STORAGE_QUERY + bool "[System-level] Enable TableStorage Buffer" + default n + + config TABLE_STORAGE_CACHE + bool "[System-level] Enable TableStorage Prefetcher" + default n + + config TABLE_STORAGE_PREFETCH + bool "[Table-level] Enable TableStorage Query Cache" + default n + + config TABLE_STORAGE_SECOND_INDEX + bool "[Table-level] Enable TableStorage Second Index" + default n + + config TABLE_STORAGE_TRANSACTION + bool "(Other) Enable TableStorage Transaction" + default n + + +endif + +endmenu diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/README.md b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/README.md new file mode 100644 index 000000000..f08c33635 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/README.md @@ -0,0 +1,153 @@ +## README + +TableStorage是一款面向泛在操作系统的轻量级表存储原型系统。TableStorage专注于泛在操作系统场景下结构化数据的存储，与传统的多层堆叠式软件栈（数据库 + 文件系统）不同，TableStorage避免过度分层，从存储全栈角度，进行跨层（数据库的存储引擎 + 文件系统）的设计，主要包含以下三个属性 + +- 低冗余：去除文件抽象并直接将“表”存储到设备中, 避免功能冗余和不必要的软件开销 +- 兼容性：提供一组通用的API 以支持表级的存取操作，与传统数据库中的读写操作兼容 +- 可集成：支持组件的深度集成，具体来说, 可以集成事务和执行引擎以满足复杂的事务和查询处理需求 + + + +### 开发板 + +- K210最小系统板（Max bit） + +- SD卡配置 + + | 引脚 | 作用 | RW007板子 | + | :----------------: | :-------: | :-------: | + | io 27(印丝标注SCK) | SPI1_SCK | SCK | + | io 26(印丝标注SO) | SPI1_MISO | MISO | + | io 28(印丝标注SI) | SPI1_MOSI | MOSI | + | io 29 | CS/BOOT1 | CS | + + + +### 编译说明 + +- 环境搭建 + + - 参考https://gitlink.org.cn/xuos/xiuos/tree/prepare_for_master/Ubiquitous%2FXiZi_IIoT%2Fboard%2Fkd233下的**开发环境搭建**小节，搭建好XiUOS的开发环境 + + - 参考https://gitlink.org.cn/xuos/xiuos/tree/prepare_for_master/Ubiquitous%2FRT-Thread_Fusion_XiUOS%2Faiit_board%2Fk210搭建好XiUOS-RTThread的开发环境 + +- 配置XiUOS-RTThread基本环境 + + - SD卡的配置：按照上表SD卡引脚说明配置 + + image-20221123215723700 + + - 其他推荐配置 + + image-20221123215509060 + + - （可选）如果编译时，出现定时器错误，则可以选择使用软件定时器 + + image-20221123215939427 + +- 配置TableStorage + + ```shell + scons --menuconfig + ``` + + - 若不开启TableStorage组件，则自动在SD卡上使用FATFS + + - 若开启TableStorage组件，则在SD卡上使用TableStorage，默认打开了Enable TableStorage Basic模块（注：当前版本仅支持Basic模块） + + image-20221123213344406 + +- 执行 scons 编译，若编译正确无误，在当前文件夹下生成rtthread.elf、rtthread.bin。其中rtthread.bin需要烧写到设备中进行运行 + +- 烧录及运行结果图 + + ```shell + sudo kflash -t rtthread.bin -p /dev/ttyUSB0 + ``` + + - 烧录并运行无误，则 + + image-20221123215143184 + + + +### 调试 + +- 修改k210/rtconfig.py中的BUILD选项来配置debug模式，并重新编译 + +- 安装openocd（下载ubuntu版本64位）， + + - 下载地址：[Releases · kendryte/openocd-kendryte (github.com)](https://github.com/kendryte/openocd-kendryte/releases),推荐下载地址为：http://101.36.126.201:8011/kendryte-openocd-0.2.3-ubuntu64.tar.gz + + - 安装 + + ```shell + sudo apt install libusb-dev libftdi-dev libhidapi-dev + sudo mv kendryte-openocd-0.2.3-ubuntu64.tar.gz /opt + cd /opt + sudo tar -zxvf kendryte-openocd-0.2.3-ubuntu64.tar.gz + ``` + + - 修改配置文件 + + sudo vim /opt/kendryte-openocd/tcl/k210.cfg，并复制以下内容 + + ```shell + # SiPEED USB-JTAG/TTL + interface ftdi + ftdi_device_desc "Dual RS232" + ftdi_vid_pid 0x0403 0x6010 + ftdi_layout_init 0x0508 0x0f1b + ftdi_layout_signal nTRST -data 0x0200 -noe 0x0100 + ftdi_layout_signal nSRST -data 0x0800 -noe 0x0400 + + jtag_rclk 3000 + + # server port + gdb_port 9999 + telnet_port 4444 + + # add cpu target + set _CHIPNAME riscv + jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x04e4796b + + set _TARGETNAME $_CHIPNAME.cpu + target create $_TARGETNAME riscv -chain-position $_TARGETNAME + + # command + init + if {[ info exists pulse_srst]} { + ftdi_set_signal nSRST 0 + ftdi_set_signal nSRST 1 + ftdi_set_signal nSRST z + } + halt + ``` + +- 调试器和Max bit开发板的硬件连线 + + image-20221123220234259 + +- 启动调试器 + + ```shell + sudo /opt/kendryte-openocd/bin/openocd -f /opt/kendryte-openocd/tcl/k210.cfg + ``` + + image-20221123221828637 + +- 在Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210目录下，连接调试器 + + ```shell + /opt/xpack-riscv-none-embed-gcc-10.2.0-1.2/bin/riscv-none-embed-gdb rtthread.elf --eval-command="target remote 127.0.0.1:9999" + ``` + + - gdb终端调试 + + image-20221123222125802 + +image-20221123223127817 + +- vscode调试 + + image-20221123225705614 diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/SConscript b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/SConscript new file mode 100644 index 000000000..f4118ef8b --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/SConscript @@ -0,0 +1,26 @@ +from building import * + +# get current directory +cwd = GetCurrentDir() + +# the set of source files associated with this SConscript file. +src = Glob('src/common/*.cc') +src += Glob('src/execution/*.cc') +src += Glob('src/storage/*.cc') + +# compile optional modules +if GetDepend(['TABLE_STORAGE_CACHE']): + src += ['src/modules/prefetcher.cc'] +if GetDepend(['TABLE_STORAGE_CACHE']): + src += ['src/modules/queryCache.cc'] +if GetDepend(['TABLE_STORAGE_CACHE']): + src += ['src/modules/buffer.cc'] +if GetDepend(['TABLE_STORAGE_CACHE']): + src += ['src/modules/secondIndex.cc'] + +# include path +path = [cwd + '/include'] + +group = DefineGroup('TableStorage', src, depend = ['TABLE_STORAGE'], CPPPATH = path) + +Return('group') \ No newline at end of file diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/BufferItem.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/BufferItem.h new file mode 100644 index 000000000..7ab6aa5dd --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/BufferItem.h @@ -0,0 +1,35 @@ +/** + * @file BufferItem.h + * @brief BufferItem + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef BUFFERITEM_H +#define BUFFERITEM_H + +#include + +#include "Common.h" + +namespace LightTable { + +class BufferItem { + public: + BufferItem(uint64_t tableID); + + virtual ~BufferItem(); + + // virtual void flush() const = 0; + + uint64_t getTableID() { return tableID; } + + protected: + uint64_t tableID; + DISALLOW_COPY_AND_ASSIGN(BufferItem); +}; + +} // namespace LightTable + +#endif // BUFFERITEM_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Common.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Common.h new file mode 100644 index 000000000..a23b31a74 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Common.h @@ -0,0 +1,144 @@ +/** + * @file Common.h + * @brief parameter configuration + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include + +#include +#include + +namespace LightTable { + +#define YCSB_TEST + +extern int cacheReadNum; +extern int cacheWriteNum; + +extern std::chrono::steady_clock::time_point start_time; +extern std::chrono::steady_clock::time_point end_time; +extern double diff_sdcard_read; +extern double diff_sdcard_write; + +#define SYSTEM_MAX_CORES 0 +#define SYSTEM_MAX_IO_QUEUES 8 + +#ifndef YCSB_TEST +#define NETWORK_MESSAGE_DATA_MAX_SIZE 4088 +#define NETWORK_TRANSMIT_MAX_SIZE 4096 +#else +#define NETWORK_MESSAGE_DATA_MAX_SIZE 51192 +#define NETWORK_TRANSMIT_MAX_SIZE 51200 +#endif + +#define SDCARD_TEST_WRITE_BLOCKID 200 +#define SDCARD_TEST_READ_BLOCKID 1000 +#define SDCARD_TEST_NUM 10000 + +#define HASH_BUCKET_SIZE 50 +#define CACHE_SIZE 500 + +#define SEGMENT_TYPE_SMALL_BITMAP_SIZE 32768 +#define SEGMENT_TYPE_MEDIUM_BITMAP_SIZE 4096 +#ifndef YCSB_TEST +#define SEGMENT_TYPE_BIG_BITMAP_SIZE 512 +#else +#define SEGMENT_TYPE_BIG_BITMAP_SIZE 512 +#endif + +#define SEGMENT_TYPE_SMALL_START 64 +#define SEGMENT_TYPE_MEDIUM_START 2097216 +#define SEGMENT_TYPE_BIG_START 6291528 + +#define SEGMENT_TYPE_SMALL_CELL_SIZE 4096 +#define SEGMENT_TYPE_MEDIUM_CELL_SIZE 65536 +#ifndef YCSB_TEST +#define SEGMENT_TYPE_BIG_CELL_SIZE 2097152 +#else +#define SEGMENT_TYPE_BIG_CELL_SIZE 65536000 +#endif + +#define PREFETCH_BLOCK_SERIALIZED_LENGTH 33 + +#define MAX_BRANCH_COUNT 4 + +#define PRELOAD_BLOCK_COUNT 10 +#define PRELOAD_CHECK_INTERVAL_US 50 + +#define BLOCK_SIZE 512 + +#ifndef YCSB_TEST +#define BUFFER_SIZE 200 +#else +#define BUFFER_SIZE 300000 +#endif + +#define CACHED_PAGE_COUNT 100 + +#define ROOTTABLE_TUPLE_SIZE sizeof(RootTable::TableTuple) + +#define TABLE_ID_SIZE sizeof(uint64_t) + +#define SYSTEM_TABLE_COUNT 1 +#define ROOTTABLE_TABLE_ID 1 +#define ROOTTABLE_SEGMENT_ID 1 +#define ROOTTABLE_FIRST_BLOCK_ID 14680164 + +#define TABLE1_META_BLOCKID 1000 +#define TABLE1_META_LOG_BLOCKID 10000 +#define TABLE1_INDEX_BLOCKID 2000 +#define TABLE1_INDEX_LOG_BLOCKID 20000 +#define TABLE1_DATA_LOG_BLOCKID 30000 +#define META_ENTRY_NUM 8 +#define META_ENTRY_SIZE 64 +#define BUFFER_MAX 4096 +#define COMMON_TABLE_START_ID 101 +#define YCSB_TABLE_ID 100 +#define TABLE_NAME_LENGTH 32 +#define PRIMARY_KEY_LENGTH 128 +#define FILE_PATH_LENGTH 1024 +#define FILE_NAME_LENGTH 128 +#define USER_NAME_LENGTH 32 +#define COLUMN_NAME_LENGTH 128 +#define COLUMN_TYPE_NAME_LENGTH 32 + +#define PREFETCH_BLOCK_SERIALIZED_LENGTH 33 +#define ROWMAP_SERIALIZED_LENGTH 42 +#define SCHEMA_ENTRY_SERIALIZED_LENGTH 181 + +#define MAX_META_LENGTH 2048 + +#define MAX_PATH_LENGTH 128 +const char METAPATH[MAX_PATH_LENGTH] = "../../data"; + +struct timespec diff(struct timespec start, struct timespec end); + +class Queue { + public: +#ifdef IO_PROFILING + static struct timespec accumulate_io_times(); + static struct timespec accumulate_io_submit_times(); +#endif + +#ifdef IO_PROFILING + static std::queue io_times; + static std::queue io_submit_times; +#endif +}; + +// A macro to disallow the copy constructor and operator= functions +#ifndef DISALLOW_COPY_AND_ASSIGN +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName &) = delete; \ + TypeName &operator=(const TypeName &) = delete; +#endif + +} // namespace LightTable + +#endif // COMMON_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/CommonTable.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/CommonTable.h new file mode 100644 index 000000000..bfc111259 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/CommonTable.h @@ -0,0 +1,35 @@ +/** + * @file CommonTable.h + * @brief CommonTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef COMMONTABLE_H +#define COMMONTABLE_H + +#include "Table.h" + +namespace LightTable { + +class CommonTable : public Table { + public: + CommonTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex = false); + + CommonTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + bool usePrimaryKeyIndex = false); + + friend class MetaHandle; + + protected: + DISALLOW_COPY_AND_ASSIGN(CommonTable); +}; + +} // namespace LightTable + +#endif // COMMONTABLE_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Driver.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Driver.h new file mode 100644 index 000000000..03d1f2945 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Driver.h @@ -0,0 +1,61 @@ +/** + * @file Driver.h + * @brief driver + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ +#ifndef DRIVER_H +#define DRIVER_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Common.h" +#include "Error.h" + +// #include +// #include +// #include + +#ifdef __cplusplus +extern "C" { +#endif +uint32_t MakeTableStorage(const char *device_name); +#ifdef __cplusplus +} +#endif + +namespace LightTable { + +class Driver { + public: + static uint32_t driver_init(const char *device_name); + + static void driver_cleanup(const char *device_name); + + static uint32_t driver_read(uint8_t *buf, uint64_t blockID); + + static uint32_t driver_write(uint8_t *buf, uint64_t blockID); + + static uint32_t read(uint8_t *buf, uint64_t blockID); + + static uint32_t write(uint8_t *buf, uint64_t blockID); + + Driver(){}; + +#ifdef TABLE_STORAGE_CACHE + static BlockCache *blkCache; +#endif +}; + +} // namespace LightTable + +#endif // DRIVER_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Error.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Error.h new file mode 100644 index 000000000..14fceb967 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Error.h @@ -0,0 +1,62 @@ +/** + * @file Error.h + * @brief error number + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef ERROR_H +#define ERROR_H + +namespace LightTable { + +enum ErrorCode { + SUCCESS = 0, + ERROR_TABLE = 1, + SEGMENT_TYPE_ERROR = 2, + SEGMENT_STATUS_ERROR = 3, + SEGMENT_IS_EXHAUSTED = 4, + BUCKET_ITEM_NOT_FOUND = 5, + ADD_BUCKET_ITEM_ERROR = 6, + PREFETCH_BLOCK_NOT_FOUND = 7, + GET_AVAILABLE_BLOCK_ERROR = 8, + BLOCK_OCCUPIED = 9, + ADD_ROW_MAP_ENTRY_ERROR = 10, + ROW_MAP_ENTRY_NOT_FOUND = 11, + ADD_SCHEMA_ENTRY_ERROR = 12, + SCHEMA_ENTRY_NOT_FOUND = 13, + COLUMN_ITEM_NOT_FOUND = 14, + COLUMN_ID_NOT_CONTINUOUS = 15, + DELETE_PARIMARY_KEY_COLUMN = 16, + TABLE_NOT_FOUND = 17, + TABLE_TUPLE_NOT_FOUND = 18, + ADD_TABLE_TUPLE_ERROR = 19, + BUFFER_EMPTY = 20, + TYPE_INVALID = 21, + ROWLOCATION_INVALID = 22, + CREATE_SOCKET_ERROR = 23, + INET_PTON_ERROR = 24, + BIND_SOCKET_ERROR = 25, + LISTEN_SOCKET_ERROR = 26, + ACCEPT_SOCKET_ERROR = 27, + CONNECT_SOCKET_ERROR = 28, + SEND_MESSAGE_ERROR = 29, + EVENT_TYPE_NOT_DEFINED = 30, + SCHEMA_NOT_EXIST = 31, + OPEN_META_FILE_ERROR = 32, + INVALID_META_FILE = 33, + INVALID_MEMORY_BLOCK_ID = 34, + UPDATE_TUPLE_ERROR = 35, + INVALID_PAGE_ID = 36, + ADD_QUERY_CACHE_ERROR = 37, + TRIE_LEAF_EXIST = 38, + TRIE_LEAF_NOT_EXIST = 39, + TRIE_LEAF_NOT_FOUND = 40, + INVALID_SCAN_RANGE = 41, + INVALID_PRELOAD_TASK = 42, +}; + +} // namespace LightTable + +#endif // ERROR_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashBucket.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashBucket.h new file mode 100644 index 000000000..620da30f2 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashBucket.h @@ -0,0 +1,59 @@ +/** + * @file HashBucket.h + * @brief HashBucket + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef HASHBUCKET_H +#define HASHBUCKET_H + +#include +// #include + +#include +#include +#include +#include +// #include + +#include "Common.h" +#include "Error.h" + +namespace LightTable { + +typedef struct rowLocation { + uint64_t blockID; + uint64_t rowOffset; +} RowLocation; + +class HashBucket { + public: + HashBucket(); + + uint32_t addBucketItem(uint64_t key, RowLocation rowLocation); + + uint32_t deleteBucketItem(uint64_t key); + + uint32_t updateRowLocation(uint64_t key, RowLocation rowLocation); + + uint32_t getRowLocation(uint64_t key, RowLocation &rowLocation); + + uint64_t getBucketSize(); + + uint32_t setBucketSize(uint64_t bucketSize); + + uint32_t getAllBucketItems(std::map &bucketItems); + + private: + std::map bucketItems; + uint64_t bucketSize; + // pthread_mutex_t mtx; + + DISALLOW_COPY_AND_ASSIGN(HashBucket); +}; + +} // namespace LightTable + +#endif // HASHBUCKET_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashTable.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashTable.h new file mode 100644 index 000000000..35cc73445 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/HashTable.h @@ -0,0 +1,63 @@ +/** + * @file HashTable.h + * @brief HashTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef HASHTABLE_H +#define HASHTABLE_H + +#include + +#include +#include +#include +#include + +#include "Common.h" +#include "HashBucket.h" + +namespace LightTable { + +class HashTable { + public: + HashTable(); + + ~HashTable(); + + uint64_t getMapKey(const char *primaryKey); + + uint32_t hashToBucket(uint64_t key); + + uint32_t getRowLocation(uint32_t bucketID, uint64_t key, + RowLocation &rowLocation); + + uint32_t addItem(uint32_t bucketID, uint64_t key, RowLocation rowLocation); + + uint32_t deleteItem(uint32_t bucketID, uint64_t key); + + uint32_t updateItem(uint32_t bucketID, uint64_t key, RowLocation rowLocation); + + uint64_t getItemCount(); + + uint32_t getAllBuckets(std::vector &buckets); + uint32_t refreshRowLocations(uint64_t blockID, uint64_t deleteOffset, + uint64_t rowSize, + std::map &locationItems); + + uint32_t clear(); + + private: + std::vector buckets; + + DISALLOW_COPY_AND_ASSIGN(HashTable); + + public: + friend class MetaHandle; +}; + +} // namespace LightTable + +#endif // HASHTABLE_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/PrefetchBlockManager.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/PrefetchBlockManager.h new file mode 100644 index 000000000..7b4ec20f8 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/PrefetchBlockManager.h @@ -0,0 +1,101 @@ +/** + * @file PrefetchBlockManager.h + * @brief PrefetchBlockManager + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef PREFETCHBLOCKMANAGER_H +#define PREFETCHBLOCKMANAGER_H + +#include +#include +#include +// #include + +#include +#include +#include +#include +#include +// #include + +#include "Error.h" +#include "Common.h" +#include "SegmentManager.h" + +namespace LightTable +{ + +enum PrefetchBlockStatus +{ + PREFETCH_BLOCK_STATUS_IDLE = 0, + PREFETCH_BLOCK_STATUS_BUSY, +}; + +class PrefetchBlockManager +{ + public: + + typedef struct prefetchBlockManagerEntry + { + uint64_t blockID; + uint64_t currentOffset; + bool isOccupied; + bool isCached; + uint64_t pageID; + }PrefetchBlockManagerEntry; + + PrefetchBlockManager(uint64_t firstBlockID, uint64_t segmentID, + SegmentType segmentType); + + ~PrefetchBlockManager(); + + uint64_t blockAllocate(uint64_t rowSize); + + uint64_t getNextBlock(); + + uint32_t setBlockState(uint64_t blockID, bool isOccupied); + + uint32_t setOffset(uint64_t blockID, uint64_t currentOffset); + + uint32_t getOffset(uint64_t blockID, uint64_t ¤tOffset); + + uint32_t advanceOffset(uint64_t blockID, uint64_t rowSize); + + uint32_t isOccupied(uint64_t blockID, bool &status); + + uint32_t getPrefetchBlockEntry(uint64_t blockID, + PrefetchBlockManagerEntry &entry); + + uint32_t getAllPrefetchBlockEntrys(std::map &entrys); + + uint32_t lockBlock(uint64_t blockID); + + uint32_t unlockBlock(uint64_t blockID); + + uint32_t setIsCached(uint64_t blockID, bool isCached, uint64_t pageID); + + bool isCached(uint64_t blockID, uint64_t &pageID); + + private: + std::map prefetchBlocks; + + const uint64_t segmentID; + const SegmentType segmentType; + const uint64_t firstBlockID; + uint64_t blockCount; + uint64_t currentID; + + DISALLOW_COPY_AND_ASSIGN(PrefetchBlockManager); + + public: + friend class QueryCache; + friend class MetaHandle; +}; + +} // namespace LightTable + +#endif // PREFETCHBLOCKMANAGER_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RWLock.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RWLock.h new file mode 100644 index 000000000..1f576b5a3 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RWLock.h @@ -0,0 +1,57 @@ +/** + * @file RWLock.h + * @brief RWLock + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ +#ifndef RWLOCK_H +#define RWLOCK_H + +#include +#include + +#include + +namespace LightTable { + +using version_lock_t = uint8_t; +#define LOAD_ORDER std::memory_order_acquire +#define STORE_ORDER std::memory_order_release + +static_assert(sizeof(version_lock_t) == 1, "Lock must be 1 byte."); +static constexpr version_lock_t CLIENT_BIT = 0b10000000; +static constexpr version_lock_t NO_CLIENT_BIT = 0b01111111; +static constexpr version_lock_t USED_BIT = 0b01000000; +static constexpr version_lock_t UNLOCKED_BIT = 0b11111110; + +#define IS_LOCKED(lock) ((lock)&1) + +class RWLock { + std::atomic version_lock; + + public: + bool lock(const bool blocking = true) { + version_lock_t lock_value = version_lock.load(LOAD_ORDER); + // Compare and swap until we are the thread to set the lock bit + lock_value &= UNLOCKED_BIT; + while (!version_lock.compare_exchange_weak(lock_value, lock_value + 1)) { + lock_value &= UNLOCKED_BIT; + if (!blocking) { + return false; + } + } + return true; + } + + void unlock() { + const version_lock_t current_version = version_lock.load(LOAD_ORDER); + version_lock_t new_version = (current_version + 1) % USED_BIT; + new_version |= USED_BIT; + new_version |= (current_version & CLIENT_BIT); + version_lock.store(new_version, STORE_ORDER); + } +}; +} // namespace LightTable + +#endif // HASHTABLE_H \ No newline at end of file diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RootTable.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RootTable.h new file mode 100644 index 000000000..8c2476635 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RootTable.h @@ -0,0 +1,75 @@ +/** + * @file RootTable.h + * @brief RootTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef ROOTTABLE_H +#define ROOTTABLE_H + +#include +#include + +#include +#include +#include + +#include "Common.h" +#include "SystemTable.h" + +namespace LightTable { + +class Buffer; + +class RootTable : public SystemTable { + public: + typedef struct tableTuple { + uint64_t tableID; + char tableName[TABLE_NAME_LENGTH]; + uint64_t firstBlockID; + uint64_t segmentID; + SegmentType segmentType; + char user[USER_NAME_LENGTH]; + struct tm time; + } TableTuple; + + RootTable(std::vector schemaEntrys, + bool usePrimaryKeyIndex) + : SystemTable(ROOTTABLE_TABLE_ID, "RootTable", ROOTTABLE_FIRST_BLOCK_ID, + ROOTTABLE_SEGMENT_ID, SEGMENT_TYPE_BIG, schemaEntrys, + usePrimaryKeyIndex) {} + + RootTable(bool usePrimaryKeyIndex = false); + + uint32_t tableNameToTableID(const char *tableName, uint64_t &tableID); + + uint32_t deleteTableInfo(uint64_t tableID); + + uint32_t appendTableInfo(TableTuple tableTuple); + + uint32_t getTableInfo(uint64_t tableID, TableTuple &tableTupleResult); + + uint32_t modifyTableInfo(uint64_t tableID, TableTuple tableTuple); + + uint32_t getTableTuple(uint64_t tableID, TableTuple &tableTuple); + + uint32_t serializeRow(const TableTuple tableTuple, uint8_t *buf); + + uint32_t deserializeRow(const uint8_t *buf, TableTuple *tableTuple); + + uint64_t getTableCount(); + + private: + std::map tableTupleMap; + + DISALLOW_COPY_AND_ASSIGN(RootTable); + + public: + friend class MetaHandle; +}; + +} // namespace LightTable + +#endif // ROOTTABLE_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RowMap.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RowMap.h new file mode 100644 index 000000000..4a14280c3 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/RowMap.h @@ -0,0 +1,57 @@ +/** + * @file RowMap.h + * @brief + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef ROWMAP_H +#define ROWMAP_H + +#include + +#include +#include +#include +#include + +#include "Common.h" +#include "Error.h" + +namespace LightTable { + +class RowMap { + public: + typedef struct rowMapEntry { + uint64_t key; + uint64_t blockID; + uint64_t rowOffset; + } RowMapEntry; + + RowMap(); + + uint32_t appendEntry(uint64_t key, uint64_t blockID, uint64_t rowOffset); + + uint32_t deleteEntry(uint64_t key); + + uint32_t alterEntry(uint64_t key, uint64_t blockID, uint64_t rowOffset); + + uint32_t queryEntry(uint64_t key, RowMapEntry &rowMapEntry); + + uint32_t queryAllEntry(std::map &allRowMapEntrys); + + uint64_t getRowCount(); + + private: + std::map rowMapEntrys; + + DISALLOW_COPY_AND_ASSIGN(RowMap); + + public: + friend class MetaHandle; +}; + +} // namespace LightTable + +#endif // ROWMAP_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Schema.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Schema.h new file mode 100644 index 000000000..051fde538 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Schema.h @@ -0,0 +1,92 @@ +/** + * @file Schema.h + * @brief table schema + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef SCHEMA_H +#define SCHEMA_H + +#include +#include + +#include +#include +#include +#include + +#include "Common.h" +#include "Error.h" + +namespace LightTable { + +class Schema { + public: + typedef struct schemaEntry { + uint64_t columnID; + char columnName[COLUMN_NAME_LENGTH]; + char type[COLUMN_TYPE_NAME_LENGTH]; + uint32_t length; + bool isPrimaryKey; + bool isEmpty; + + schemaEntry &operator=(schemaEntry &another) { + this->columnID = another.columnID; + memcpy(this->columnName, another.columnName, COLUMN_NAME_LENGTH); + memcpy(this->type, another.type, COLUMN_TYPE_NAME_LENGTH); + this->length = another.length; + this->isPrimaryKey = another.isPrimaryKey; + this->isEmpty = another.isEmpty; + + return *this; + } + + } SchemaEntry; + + Schema(); + + Schema(std::vector schemaEntrys); + + uint32_t appendEntry(SchemaEntry &schemaEntry); + + uint32_t deleteEntry(uint64_t columnID); + + uint32_t alterEntry(uint64_t columnID, SchemaEntry schemaEntry); + + uint32_t getColumnID(const char *columnName, uint64_t &columnID); + + uint32_t queryEntry(uint64_t columnID, Schema::SchemaEntry &schemaEntry); + + uint32_t queryAllEntry(std::map &allSchemaEntrys); + + uint64_t getEntrySize(); + + void setEntrySize(uint64_t entrySize); + + uint32_t getPrimaryKeyLength(); + + uint64_t getColumnCount(); + + void setColumnCount(uint64_t columnCount); + + Schema::SchemaEntry &getPrimaryKeySchema(); + + uint32_t drop(); + + private: + std::map schemaEntrys; + SchemaEntry primaryKeySchema; + uint64_t columnCount; + uint64_t entrySize; + + DISALLOW_COPY_AND_ASSIGN(Schema); + + public: + friend class MetaHandle; +}; + +} // namespace LightTable + +#endif // SCHEMA_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SegmentManager.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SegmentManager.h new file mode 100644 index 000000000..740329779 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SegmentManager.h @@ -0,0 +1,59 @@ +/** + * @file SegmentManager.h + * @brief SegmentManager + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef SEGMENTMANAGER_H +#define SEGMENTMANAGER_H + +#include + +#include "Common.h" +#include "Error.h" +#include "Driver.h" +#include "Common.h" + +namespace LightTable { + +enum SegmentType { + SEGMENT_TYPE_SMALL = 101, + SEGMENT_TYPE_MEDIUM, + SEGMENT_TYPE_BIG, +}; + +enum SegmentStatus { + SEGMENT_STATUS_IDLE = 0, + SEGMENT_STATUS_BUSY, +}; + +class SegmentManager { + public: + SegmentManager(); + ~SegmentManager(); + + uint32_t setBitmap(SegmentType segmentType, uint64_t segmentID, + SegmentStatus segmentStatus); + + uint32_t getIdleSegment(SegmentType segmentType, uint64_t &segmentID, + uint64_t &firstBlockID); + + private: + uint32_t setBit(uint64_t segmentID, uint8_t *buf, uint64_t segmentStart, + SegmentStatus segmentStatus); + + uint32_t findFirstIdle(uint8_t *buf, uint64_t segmentNum, + uint64_t &segmentID); + + uint8_t *bigBitmap; + uint8_t *mediumBitmap; + uint8_t *smallBitmap; + + DISALLOW_COPY_AND_ASSIGN(SegmentManager); +}; + +} // namespace LightTable + +#endif // SEGMENTMANAGER_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SystemTable.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SystemTable.h new file mode 100644 index 000000000..d3f19b360 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/SystemTable.h @@ -0,0 +1,33 @@ +/** + * @file SystemTable.h + * @brief SystemTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef SYSTEMTABLE_H +#define SYSTEMTABLE_H + +#include "Table.h" + +namespace LightTable { + +class SystemTable : public Table { + public: + SystemTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex); + + SystemTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + bool usePrimaryKeyIndex); + + protected: + DISALLOW_COPY_AND_ASSIGN(SystemTable); +}; + +} // namespace LightTable + +#endif // SYSTEMTABLE_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Table.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Table.h new file mode 100644 index 000000000..17d906f99 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/Table.h @@ -0,0 +1,147 @@ +/** + * @file Table.h + * @brief Table + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef TABLE_H +#define TABLE_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "Common.h" +#include "HashTable.h" +#include "RowMap.h" +#include "Schema.h" +#include "Driver.h" +#include "UndoLogEntry.h" +#include "SegmentManager.h" +#include "PrefetchBlockManager.h" + +namespace LightTable { +#ifdef TABLE_STORAGE_SECOND_INDEX + // TODO: second index +#endif + +class Buffer; + +class Table { + typedef struct currentLocation { + uint64_t blockID; + int32_t emptySlotNum; + } currentLocation; + + typedef struct metaLogEntry { + uint64_t tableID; + uint64_t tupleNum; + std::time_t t; + } metaLogEntry; + + public: + Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex = false); + + Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + bool usePrimaryKeyIndex = false); + + uint32_t drop(); + + uint32_t truncate(); + + uint32_t insertRow(const uint8_t *rowData); + + uint32_t insertRow_logging(const uint8_t *rowData); + + uint32_t deleteRow(const char *primaryKey); + + uint32_t updateRow(const char *primaryKey, const uint8_t *rowData); + + uint32_t updateColumnItem(const char *primaryKey, const char *columnName, + const uint8_t *columnData); + + uint32_t updateColumnItem_logging(const char *primaryKey, + const char *columnName, + const uint8_t *columnData); + + uint32_t selectRow(const char *primaryKey, uint8_t *rowData); + + uint32_t selectColumnItem(const char *primaryKey, const char *columnName, + uint8_t *columnData); + + uint64_t getRowCount(); + + uint64_t getTableID(); + + char *getTableName(); + + uint32_t setTableID(uint64_t tableID); + + uint32_t setTableName(const char *tableName); + + Schema &getSchema(); + + HashTable &getHashTable(); + + protected: + uint32_t getColumnID(const char *columnName, uint64_t &columnID); + + uint32_t parsePrimaryKey(const uint8_t *rowData, char *primaryKeyData); + + uint32_t locateRow(const char *primaryKey, RowLocation &rowLocation); + + uint32_t getRowFromBlock(const char *primaryKey, uint8_t *rowData, + const uint8_t *blockData, + const RowLocation rowLocation); + + uint32_t selectRow(const char *primaryKey, uint8_t *rowData, + RowLocation &rowLocation); + + uint32_t deleteRowData(uint64_t key, RowLocation rowLocation); + + uint32_t getColumnOffset(const char *columnName, uint64_t &columnID, + uint32_t &columnLength, uint64_t &offset); + + uint64_t tableID; + char tableName[TABLE_NAME_LENGTH]; + Schema schema; + RowMap rowMap; + HashTable hashTable; + bool usePrimaryKeyIndex; + uint64_t tupleNum; + std::time_t t; + currentLocation indexLocation; + currentLocation metaLogLocation; + currentLocation indexLogLocation; + currentLocation dataLogLocation; + uint8_t indexData[BLOCK_SIZE]; + uint8_t metaData[BLOCK_SIZE]; + uint8_t metaLog[BLOCK_SIZE]; + uint8_t indexLog[BLOCK_SIZE]; + uint8_t dataLog[BLOCK_SIZE]; + PrefetchBlockManager prefetchBlockManager; + + DISALLOW_COPY_AND_ASSIGN(Table); + + public: + friend class DaemonProcess; + friend class MetaHandle; + friend class QueryCache; +}; + +} // namespace LightTable + +#endif // TABLE_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/TableStorage.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/TableStorage.h new file mode 100644 index 000000000..d81d14082 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/TableStorage.h @@ -0,0 +1,145 @@ +/** + * @file TableStorage.h + * @brief TableStorage + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef DAEMONPROCESS_H +#define DAEMONPROCESS_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// #include +// #include +// #include +// #include +// #include + +#include "RootTable.h" +#include "SegmentManager.h" +#include "Common.h" +#include "CommonTable.h" +#include "Error.h" +#include "Driver.h" +#include "Table.h" + +#define TABLE_DEBUG_ENABLE +#define PROJECT_NAME "TanleStorage" + +#ifndef TABLE_PRINT +#define TABLE_PRINT(...) printf(__VA_ARGS__) +#endif +#define TABLE_LOG_PREFIX1() \ + TABLE_PRINT("[\e[1;34m%s\033[0m] \e[1;31m%s\033[0m", PROJECT_NAME, \ + __FUNCTION__) +#define TABLE_LOG_PREFIX2() TABLE_PRINT(" ") +#define TABLE_LOG_PREFIX() \ + TABLE_LOG_PREFIX1(); \ + TABLE_LOG_PREFIX2() + +#ifdef TABLE_DEBUG_ENABLE +#define TABLE_DEBUG(...) \ + TABLE_LOG_PREFIX(); \ + TABLE_PRINT("(\e[1;32m%s:%d\033[0m) ", __FILE__, __LINE__); \ + TABLE_PRINT(__VA_ARGS__) +#else +#define TABLE_DEBUG(...) +#endif + +namespace LightTable { + +enum EventType { + CREATE_TABLE_EVENT = 10001, + INSERT_ROW_EVENT, + DELETE_ROW_EVENT, + SELECT_ROW_EVENT, +#ifdef YCSB_TEST + SCAN_EVENT, +#endif + UPDATE_ROW_EVENT, +#ifdef YCSB_TEST + UPDATE_COLUMNS_EVENT, +#endif + DROP_TABLE_EVENT, + RESPOND_CLIENT_REQUEST, + STOP_SERVICE_EVENT, +#ifdef YCSB_TEST + UNKNOWN_EVENT, +#endif +}; + +typedef struct message { + uint32_t size; + EventType eventType; + uint8_t data[NETWORK_MESSAGE_DATA_MAX_SIZE]; +} Message; + +class TableStorage { + public: + static TableStorage *getInstance(const char *ipAddress, + const uint32_t portNum, + const char *path = METAPATH) { + static TableStorage *daemonProcess; + + // To determine whether the first call. + if (daemonProcess == NULL) { + daemonProcess = new TableStorage(ipAddress, portNum, path); + } + + return daemonProcess; + } + + uint32_t createTable(const char *tableName, SegmentType segmentType, + std::vector &schema); + + uint32_t dropTable(const char *tableName); + + uint32_t insertTuple(const char *tableName, const uint8_t *row); + + uint32_t deleteTuple(const char *tableName, const char *primaryKey); + + uint32_t updateTuple(const char *tableName, const char *primaryKey, + const char *columnName, const uint8_t *value); + + uint32_t selectTuple(const char *tableName, const char *primaryKey, + uint8_t *row); + + uint32_t initTableStorage(); + uint32_t closeTableStorage(); + + private: + TableStorage(const char *ipAddress, const uint32_t portNum, const char *path); + ~TableStorage(); + + RootTable *rootTable; + std::map tableMap; + SegmentManager *segmentManager; + const char *LightTableIPAddress; + const uint32_t LightTablePortNum; + + // int32_t socketfd; + // MetaHandle metaHandle; + // Buffer *buf; + // QueryCache *queryCache; + // BlockPreload *blockPreload; + + friend class Benchmark; +}; + +} // namespace LightTable + +#endif // DAEMONPROCESS_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/ThreadSafeQueue.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/ThreadSafeQueue.h new file mode 100644 index 000000000..5ad130b6f --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/ThreadSafeQueue.h @@ -0,0 +1,81 @@ +/** + * @file ThreadSafeQueue.h + * @brief ThreadSafeQueue + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include + +#include + +#include "Common.h" + +using namespace std; + +namespace LightTable { + +template +class ThreadSafeQueue { + private: + pthread_mutex_t m_lock; + int m_front; + int m_rear; + object m_data[BUFFER_SIZE]; + + public: + ThreadSafeQueue() : m_front(0), m_rear(0) { + pthread_mutex_init(&m_lock, NULL); + } + + bool EnQueue(object data) { + pthread_mutex_lock(&m_lock); + if (isFull()) { + // cout<<"The queue is full!"< + +#include "BufferItem.h" +#include "Common.h" +#include "Error.h" + +namespace LightTable { + +class UndoLogEntry : public BufferItem { + public: + UndoLogEntry(uint64_t tableID, const char *primaryKey, uint64_t keySize, + char *tuple, uint64_t tupleSize, uint64_t blockID, + uint64_t offset); + + static bool getIsExpired(UndoLogEntry *logEntry); + void expireUndoLog(); + + uint32_t getTuple(uint64_t tableID, const char *primaryKey, char *tuple, + uint64_t &blockID, uint64_t &offset); + + char *getPrimaryKey(); + + ~UndoLogEntry(); + + private: + char *tuple; + char *primaryKey; + uint64_t blockID; + uint64_t offset; + uint64_t tupleSize; + uint64_t keySize; + bool isExpired; + DISALLOW_COPY_AND_ASSIGN(UndoLogEntry); +}; + +} // namespace LightTable + +#endif // UNDOLOGENTRY_H diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/integer.h b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/integer.h new file mode 100644 index 000000000..51b560b35 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/include/integer.h @@ -0,0 +1,41 @@ +/** + * @file integer.h + * @brief integer + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#ifndef _FF_INTEGER +#define _FF_INTEGER + +#ifdef _WIN32 /* FatFs development platform */ + +#include +#include +typedef unsigned __int64 QWORD; + +#else /* Embedded platform */ + +/* These types MUST be 16-bit or 32-bit */ +typedef int INT; +typedef unsigned int UINT; + +/* This type MUST be 8-bit */ +typedef unsigned char BYTE; + +/* These types MUST be 16-bit */ +typedef short SHORT; +typedef unsigned short WORD; +typedef unsigned short WCHAR; + +/* These types MUST be 32-bit */ +typedef long LONG; +typedef unsigned long DWORD; + +/* This type MUST be 64-bit (Remove this for C89 compatibility) */ +typedef unsigned long long QWORD; + +#endif + +#endif diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123213344406.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123213344406.png new file mode 100644 index 000000000..d84f32b7c Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123213344406.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215143184.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215143184.png new file mode 100644 index 000000000..9cce6c818 Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215143184.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215509060.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215509060.png new file mode 100644 index 000000000..0e65521af Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215509060.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215723700.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215723700.png new file mode 100644 index 000000000..00d9c63ce Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215723700.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215939427.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215939427.png new file mode 100644 index 000000000..8ce855731 Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123215939427.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123220234259.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123220234259.png new file mode 100644 index 000000000..7250d5b6e Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123220234259.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123221828637.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123221828637.png new file mode 100644 index 000000000..2cde14d8b Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123221828637.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123222125802.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123222125802.png new file mode 100644 index 000000000..448670d67 Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123222125802.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123223127817.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123223127817.png new file mode 100644 index 000000000..42b8c8164 Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123223127817.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123225705614.png b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123225705614.png new file mode 100644 index 000000000..c8dea4253 Binary files /dev/null and b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/pic/image-20221123225705614.png differ diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Common.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Common.cc new file mode 100644 index 000000000..e3d1d3d77 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Common.cc @@ -0,0 +1,80 @@ +/** + * @file Common.cc + * @brief Common + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "Common.h" + +namespace LightTable { + +int cacheReadNum = 0; +int cacheWriteNum = 0; + +std::chrono::steady_clock::time_point start_time = + std::chrono::steady_clock::now(); +std::chrono::steady_clock::time_point end_time = + std::chrono::steady_clock::now(); +double diff_sdcard_read = 0.0; +double diff_sdcard_write = 0.0; + +struct timespec diff(struct timespec start, struct timespec end) { + struct timespec temp; + if ((end.tv_nsec - start.tv_nsec) < 0) { + temp.tv_sec = end.tv_sec - start.tv_sec - 1; + temp.tv_nsec = 1000000000 + end.tv_nsec - start.tv_nsec; + } else { + temp.tv_sec = end.tv_sec - start.tv_sec; + temp.tv_nsec = end.tv_nsec - start.tv_nsec; + } + return temp; +} + +#ifdef IO_PROFILING +std::queue Queue::io_times; +std::queue Queue::io_submit_times; +#endif + +#ifdef IO_PROFILING +struct timespec Queue::accumulate_io_times() { + struct timespec sum; + sum.tv_sec = 0; + sum.tv_nsec = 0; + + while (!io_times.empty()) { + sum.tv_sec += io_times.front().tv_sec; + if (1000000000 <= sum.tv_nsec + io_times.front().tv_nsec) { + sum.tv_sec++; + sum.tv_nsec = sum.tv_nsec + io_times.front().tv_nsec - 1000000000; + } else { + sum.tv_nsec += io_times.front().tv_nsec; + } + io_times.pop(); + } + + return sum; +} + +struct timespec Queue::accumulate_io_submit_times() { + struct timespec sum; + sum.tv_sec = 0; + sum.tv_nsec = 0; + + while (!io_submit_times.empty()) { + sum.tv_sec += io_submit_times.front().tv_sec; + if (1000000000 <= sum.tv_nsec + io_submit_times.front().tv_nsec) { + sum.tv_sec++; + sum.tv_nsec = sum.tv_nsec + io_submit_times.front().tv_nsec - 1000000000; + } else { + sum.tv_nsec += io_submit_times.front().tv_nsec; + } + io_submit_times.pop(); + } + + return sum; +} +#endif + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Driver.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Driver.cc new file mode 100644 index 000000000..2774936a4 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Driver.cc @@ -0,0 +1,92 @@ +/** + * @file Driver.cc + * @brief Driver + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "Driver.h" + +extern "C" { + #include "ff.h" + #include "ffconf.h" + + #include + #include +} + +/* Definitions of physical drive number for each drive */ +#define DEV_RAM 0 /* Example: Map Ramdisk to physical drive 0 */ +#define DEV_MMC 1 /* Example: Map MMC/SD card to physical drive 1 */ +#define DEV_USB 2 /* Example: Map USB MSD to physical drive 2 */ + +static rt_device_t disk[FF_VOLUMES] = {0}; + +namespace LightTable { + +uint32_t Driver::write(uint8_t *buf, uint64_t blockID) { + rt_size_t result; + rt_device_t device = disk[0]; + +#ifdef TABLE_STORAGE_CACHE + // TODO: cache +#endif + + if (rt_device_write(device, blockID, buf, 1) == 1) + return RES_OK; + else + return RES_ERROR; +} + +uint32_t Driver::read(uint8_t *buf, uint64_t blockID) { + rt_size_t result; + rt_device_t device = disk[0]; + +#ifdef TABLE_STORAGE_CACHE + // TODO: cache +#endif + + if (rt_device_read(device, blockID, buf, 1) == 1) { + return RES_OK; + } + + return RES_OK; +} + +void Driver::driver_cleanup(const char *device_name) { + /* clean device */ + return; +} + +uint32_t Driver::driver_init(const char *device_name) { + /* init device */ + /* note: initialization of the device is done at the MakeTableStorage*/ + return 0; +} +} // namespace LightTable + +uint32_t MakeTableStorage(const char *device_name) { + rt_device_t dev_id; + + /* open specific device */ + if (device_name == NULL) { + /* which is a non-device filesystem mount */ + dev_id = NULL; + } else if ((dev_id = rt_device_find(device_name)) == NULL) { + /* no this device */ + rt_set_errno(-ENODEV); + return -1; + } + + /* open device, but do not check the status of device */ + if (dev_id != NULL) { + if (rt_device_open(dev_id, RT_DEVICE_OFLAG_RDWR) != RT_EOK) { + return -1; + } + } + + disk[0] = dev_id; + + return 0; +} diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Makefile b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Makefile new file mode 100644 index 000000000..c5b1f927f --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/common/Makefile @@ -0,0 +1,22 @@ +ifndef LIGHTTABLE_DIR +LIGHTTABLE_DIR = ../.. +endif + +SRC_FILES := $(wildcard *.cc) +INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I. -I$(LIGHTTABLE_DIR)/spdk/include + +SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1))) + +CXXFLAGS += -Werror -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter \ + -Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \ + $(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \ + -fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \ + -D_FORTIFY_SOURCE=2 -std=c++11 + +CXXFLAGS += -DYCSB_TEST + +CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \ + -Wl,-z,noexecstack \ + -Wl,--whole-archive -Wl,--no-whole-archive + +include $(KERNEL_ROOT)/compiler.mk \ No newline at end of file diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/BufferItem.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/BufferItem.cc new file mode 100644 index 000000000..fa2de2f8b --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/BufferItem.cc @@ -0,0 +1,17 @@ +/** + * @file BufferItem.cc + * @brief BufferItem + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "BufferItem.h" + +namespace LightTable { + +BufferItem::BufferItem(uint64_t tableID) : tableID(tableID) {} + +BufferItem::~BufferItem() {} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/Makefile b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/Makefile new file mode 100644 index 000000000..1e1a1c1f4 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/Makefile @@ -0,0 +1,23 @@ +ifndef LIGHTTABLE_DIR +LIGHTTABLE_DIR = ../.. +endif + +SRC_FILES := $(wildcard *.cc) +INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I. + +SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1))) + +CXXFLAGS += -Werror -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter \ + -Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \ + $(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \ + -fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \ + -D_FORTIFY_SOURCE=2 -std=c++11 + +CXXFLAGS += -DYCSB_TEST + +CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \ + -Wl,-z,noexecstack \ + -Wl,--whole-archive -Wl,--no-whole-archive + + +include $(KERNEL_ROOT)/compiler.mk diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/TableStorage.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/TableStorage.cc new file mode 100644 index 000000000..5ba3542fe --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/TableStorage.cc @@ -0,0 +1,184 @@ +/** + * @file TableStorage.cc + * @brief TableStorage + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "TableStorage.h" + +// #include "ThreadPool.h" +// #include "Buffer.h" + +// static struct timespec time1, time2, time3; +// static uint32_t coreNum = 0; + +namespace LightTable { + +TableStorage::TableStorage(const char *ipAddress, const uint32_t portNum, + const char *path) + : // metaHandle(path), + LightTableIPAddress(ipAddress), + LightTablePortNum(portNum) { + + segmentManager = new SegmentManager(); + rootTable = new RootTable(); + + std::cout << "INIT TableStorage SUCCESS" << "\n"; + // threadPool = ThreadPool::getInstance(SYSTEM_MAX_CORES); + // buf = Buffer::getInstance(); + // queryCache = QueryCache::getInstance(&tableMap); + // blockPreload = BlockPreload::getInstance(); + // blockPreload->setParameters(queryCache); + // buf->setParameters(&metaHandle, rootTable); +} + +uint32_t TableStorage::createTable(const char *tableName, + SegmentType segmentType, + std::vector &schema) { + uint64_t segmentID = 0; + uint64_t firstBlockID = 0; + uint64_t tableID = + tableMap.size() + COMMON_TABLE_START_ID - SYSTEM_TABLE_COUNT; + uint32_t err = + segmentManager->getIdleSegment(segmentType, segmentID, firstBlockID); + if (err != SUCCESS) { + return err; + } + + CommonTable *commonTable = new CommonTable(tableID, tableName, firstBlockID, + segmentID, segmentType, schema); + std::pair::iterator, bool> ret; + ret = tableMap.insert(std::pair(tableID, commonTable)); + if (false == ret.second) { + return ERROR_TABLE; + } + time_t timeNow = time(NULL); + struct tm *now = localtime(&timeNow); + RootTable::TableTuple tuple; + tuple.tableID = tableID; + memcpy(tuple.tableName, tableName, TABLE_NAME_LENGTH); + tuple.firstBlockID = firstBlockID; + tuple.segmentType = segmentType; + tuple.segmentID = segmentID; + memcpy(tuple.user, "user", 5); + tuple.time = *now; + err = rootTable->appendTableInfo(tuple); + + if (err == SUCCESS) std::cout << "create table: " << tableName << std::endl; + return err; +} + +uint32_t TableStorage::dropTable(const char *tableName) { + uint64_t tableID = 0; + uint32_t result = rootTable->tableNameToTableID(tableName, tableID); + if (SUCCESS != result) return result; + + std::map::iterator iter = tableMap.find(tableID); + Table *commonTable = iter->second; + commonTable->drop(); + tableMap.erase(tableID); + result = rootTable->deleteTableInfo(tableID); + + if (result == SUCCESS) std::cout << "drop table: " << tableName << std::endl; + return result; +} + +uint32_t TableStorage::insertTuple(const char *tableName, const uint8_t *row) { + uint64_t tableID = 0; + uint32_t result = 0; + +#ifndef SDCARD_TEST + result = rootTable->tableNameToTableID(tableName, tableID); + if (SUCCESS != result) { + return result; + } +#else + tableID = YCSB_TABLE_ID; +#endif + + std::map::iterator iter = tableMap.find(tableID); + Table *commonTable = iter->second; + +#ifndef TABLE_STORAGE_LOGGING + result = commonTable->insertRow(row); +#else + result = commonTable->insertRow_logging(row); +#endif + + return result; +} + +uint32_t TableStorage::deleteTuple(const char *tableName, + const char *primaryKey) { + uint64_t tableID = 0; + uint32_t result = rootTable->tableNameToTableID(tableName, tableID); + if (SUCCESS != result) return result; + + std::map::iterator iter = tableMap.find(tableID); + Table *commonTable = iter->second; + result = commonTable->deleteRow(primaryKey); + + if (result == SUCCESS) + std::cout << "delete a tuple in " << tableName << " success!" << std::endl; + return result; +} + +uint32_t TableStorage::updateTuple(const char *tableName, + const char *primaryKey, + const char *columnName, + const uint8_t *value) { + uint64_t tableID = 0; + uint32_t result = rootTable->tableNameToTableID(tableName, tableID); + if (SUCCESS != result) return result; + + std::map::iterator iter = tableMap.find(tableID); + Table *commonTable = iter->second; + +#ifndef TABLE_STORAGE_LOGGING + result = commonTable->updateColumnItem(primaryKey, columnName, value); +#else + result = commonTable->updateColumnItem_logging(primaryKey, columnName, value); +#endif + + return result; +} + +uint32_t TableStorage::selectTuple(const char *tableName, + const char *primaryKey, uint8_t *row) { + uint64_t tableID = 0; + uint32_t result = 0; + +#ifndef SDCARD_TEST + result = rootTable->tableNameToTableID(tableName, tableID); + if (SUCCESS != result) return result; +#else + tableID = YCSB_TABLE_ID; +#endif + + std::map::iterator iter = tableMap.find(tableID); + Table *commonTable = iter->second; + result = commonTable->selectRow(primaryKey, row); + + return result; +} + +uint32_t TableStorage::initTableStorage() { + return SUCCESS; +} + +uint32_t TableStorage::closeTableStorage() { + return SUCCESS; +} + +TableStorage::~TableStorage() { + for (auto ite = tableMap.begin(); ite != tableMap.end(); ++ite) + delete (ite->second); + tableMap.clear(); + delete (rootTable); + delete (segmentManager); + std::cout << "close TableStorage success!\n"; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/UndoLogEntry.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/UndoLogEntry.cc new file mode 100644 index 000000000..b43735397 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/execution/UndoLogEntry.cc @@ -0,0 +1,54 @@ +/** + * @file UndoLogEntry.cc + * @brief UndoLogEntry + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "UndoLogEntry.h" + +#include + +namespace LightTable { + +UndoLogEntry::UndoLogEntry(uint64_t tableID, const char *primaryKey, + uint64_t keySize, char *tuple, uint64_t tupleSize, + uint64_t blockID, uint64_t offset) + : BufferItem(tableID), + blockID(blockID), + offset(offset), + tupleSize(tupleSize), + keySize(keySize), + isExpired(false) { + memcpy(this->tuple, tuple, tupleSize); + memcpy(this->primaryKey, primaryKey, keySize); +} + +UndoLogEntry::~UndoLogEntry() { + delete[] primaryKey; + delete[] tuple; +} + +uint32_t UndoLogEntry::getTuple(uint64_t tableID, const char *primaryKey, + char *tuple, uint64_t &blockID, + uint64_t &offset) { + if (this->tableID != tableID || strcmp(this->primaryKey, primaryKey) != 0) { + return ERROR_TABLE; + } + + memcpy(tuple, this->tuple, tupleSize); + blockID = this->blockID; + offset = this->offset; + return SUCCESS; +} + +void UndoLogEntry::expireUndoLog() { isExpired = true; } + +char *UndoLogEntry::getPrimaryKey() { return this->primaryKey; } + +bool UndoLogEntry::getIsExpired(UndoLogEntry *logEntry) { + return logEntry->isExpired; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/CommonTable.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/CommonTable.cc new file mode 100644 index 000000000..1a7eb8ea5 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/CommonTable.cc @@ -0,0 +1,31 @@ +/** + * @file CommonTable.cc + * @brief CommonTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "CommonTable.h" + +namespace LightTable { + +CommonTable::CommonTable(uint64_t tableID, const char *tableName, + uint64_t firstBlockID, uint64_t segmentID, + SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex) + : Table(tableID, tableName, firstBlockID, segmentID, segmentType, + schemaEntrys, usePrimaryKeyIndex) + +{} + +CommonTable::CommonTable(uint64_t tableID, const char *tableName, + uint64_t firstBlockID, uint64_t segmentID, + SegmentType segmentType, bool usePrimaryKeyIndex) + : Table(tableID, tableName, firstBlockID, segmentID, segmentType, + usePrimaryKeyIndex) + +{} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashBucket.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashBucket.cc new file mode 100644 index 000000000..a32852020 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashBucket.cc @@ -0,0 +1,77 @@ +/** + * @file HashBucket.cc + * @brief HashBucket + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "HashBucket.h" + +namespace LightTable { + +HashBucket::HashBucket() : bucketSize(0) {} + +uint32_t HashBucket::addBucketItem(uint64_t key, RowLocation rowLocation) { + std::pair::iterator, bool> ret; + // std::unique_lock lck(mtx); + // pthread_mutex_lock(&mtx); + ret = bucketItems.insert(std::pair(key, rowLocation)); + // lck.unlock(); + // pthread_mutex_unlock(&mtx); + if (ret.second == false) { + return ADD_BUCKET_ITEM_ERROR; + } + bucketSize++; + return SUCCESS; +} + +uint32_t HashBucket::deleteBucketItem(uint64_t key) { + if (bucketItems.erase(key) == 1) { + bucketSize--; + return SUCCESS; + } + return BUCKET_ITEM_NOT_FOUND; +} + +uint32_t HashBucket::updateRowLocation(uint64_t key, RowLocation rowLocation) { + std::map::iterator iter; + iter = bucketItems.find(key); + + if (iter != bucketItems.end()) { + (iter->second).blockID = rowLocation.blockID; + (iter->second).rowOffset = rowLocation.rowOffset; + return SUCCESS; + } + return BUCKET_ITEM_NOT_FOUND; +} + +uint32_t HashBucket::getRowLocation(uint64_t key, RowLocation &rowLocation) { + std::map::iterator iter; + iter = bucketItems.find(key); + + if (iter != bucketItems.end()) { + rowLocation.blockID = (iter->second).blockID; + rowLocation.rowOffset = (iter->second).rowOffset; + return SUCCESS; + } + + rowLocation.blockID = 0; + rowLocation.rowOffset = 0; + return BUCKET_ITEM_NOT_FOUND; +} + +uint64_t HashBucket::getBucketSize() { return bucketSize; } + +uint32_t HashBucket::setBucketSize(uint64_t bucketSize) { + this->bucketSize = bucketSize; + return SUCCESS; +} + +uint32_t HashBucket::getAllBucketItems( + std::map &bucketItems) { + bucketItems = this->bucketItems; + return SUCCESS; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashTable.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashTable.cc new file mode 100644 index 000000000..bbb50dd78 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/HashTable.cc @@ -0,0 +1,113 @@ +/** + * @file HashTable.cc + * @brief HashTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "HashTable.h" + +namespace LightTable { + +HashTable::HashTable() { + buckets.reserve(HASH_BUCKET_SIZE); + + for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) { + HashBucket *tempBucket = new HashBucket(); + buckets.push_back(tempBucket); + } +} + +HashTable::~HashTable() { + std::vector::iterator iter = buckets.begin(); + while (iter != buckets.end()) { + iter = buckets.erase(iter); + } +} + +uint64_t HashTable::getMapKey(const char *primaryKey) { + const std::string str = primaryKey; + + uint64_t BitsInUnsignedInt = (uint64_t)(4 * 8); + uint64_t ThreeQuarters = (uint64_t)((BitsInUnsignedInt * 3) / 4); + uint64_t OneEighth = (uint64_t)(BitsInUnsignedInt / 8); + uint64_t HighBits = (uint64_t)(0xFFFFFFFF) << (BitsInUnsignedInt - OneEighth); + uint64_t hash = 0; + uint64_t test = 0; + for (uint32_t i = 0; i < str.length(); i++) { + hash = (hash << OneEighth) + str[i]; + if ((test = hash & HighBits) != 0) { + hash = ((hash ^ (test >> ThreeQuarters)) & (~HighBits)); + } + } + return hash; +} + +uint32_t HashTable::hashToBucket(uint64_t key) { + return key % HASH_BUCKET_SIZE; +} + +uint32_t HashTable::getRowLocation(uint32_t bucketID, uint64_t key, + RowLocation &rowLocation) { + return buckets[bucketID]->getRowLocation(key, rowLocation); +} + +uint32_t HashTable::addItem(uint32_t bucketID, uint64_t key, + RowLocation rowLocation) { + return buckets[bucketID]->addBucketItem(key, rowLocation); +} + +uint32_t HashTable::deleteItem(uint32_t bucketID, uint64_t key) { + return buckets[bucketID]->deleteBucketItem(key); +} + +uint32_t HashTable::updateItem(uint32_t bucketID, uint64_t key, + RowLocation rowLocation) { + return buckets[bucketID]->updateRowLocation(key, rowLocation); +} + +uint64_t HashTable::getItemCount() { + if (buckets.size() == 0) { + return buckets.size(); + } + uint64_t itemCount = 0; + for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) { + itemCount += buckets[i]->getBucketSize(); + } + return itemCount; +} + +uint32_t HashTable::getAllBuckets(std::vector &buckets) { + buckets = this->buckets; + return SUCCESS; +} + +uint32_t HashTable::refreshRowLocations( + uint64_t blockID, uint64_t deleteOffset, uint64_t rowSize, + std::map &locationItems) { + for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) { + std::map bucketItems; + buckets[i]->getAllBucketItems(bucketItems); + std::map::iterator iter = bucketItems.begin(); + while (iter != bucketItems.end()) { + if ((iter->second).blockID == blockID && + (iter->second).rowOffset > deleteOffset) { + (iter->second).rowOffset -= rowSize; + locationItems.insert( + std::pair(iter->first, iter->second)); + } + } + } + return SUCCESS; +} + +uint32_t HashTable::clear() { + std::vector::iterator iter = buckets.begin(); + while (iter != buckets.end()) { + iter = buckets.erase(iter); + } + return SUCCESS; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Makefile b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Makefile new file mode 100644 index 000000000..13f5df2ac --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Makefile @@ -0,0 +1,23 @@ +ifndef LIGHTTABLE_DIR +LIGHTTABLE_DIR = ../.. +endif + +SRC_FILES := $(wildcard *.cc) +INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I. -I$(LIGHTTABLE_DIR)/spdk/include -I/opt/gnu-mcu-eclipse/riscv-none-gcc/8.2.0-2.1-20190425-1021/riscv-none-embed/include/c++/8.2.0 + +SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1))) + +CXXFLAGS += -fno-omit-frame-pointer \ + -Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \ + $(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \ + -fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \ + -D_FORTIFY_SOURCE=2 -std=c++11 + +CXXFLAGS += -DYCSB_TEST + +CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \ + -Wl,-z,noexecstack \ + -Wl,--whole-archive -Wl,--no-whole-archive -lrt + + +include $(KERNEL_ROOT)/compiler.mk diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/PrefetchBlockManager.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/PrefetchBlockManager.cc new file mode 100644 index 000000000..53d968529 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/PrefetchBlockManager.cc @@ -0,0 +1,267 @@ +/** + * @file PrefetchBlockManager.cc + * @brief PrefetchBlockManager + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "PrefetchBlockManager.h" + +namespace LightTable +{ + +PrefetchBlockManager::PrefetchBlockManager(uint64_t firstBlockID, + uint64_t segmentID, + SegmentType segmentType) + : segmentID(segmentID), segmentType(segmentType), firstBlockID(firstBlockID), currentID(0) +{ + uint32_t num = 0; + + switch (segmentType) + { + case SEGMENT_TYPE_SMALL: + num = SEGMENT_TYPE_SMALL_CELL_SIZE / BLOCK_SIZE; + break; + case SEGMENT_TYPE_MEDIUM: + num = SEGMENT_TYPE_MEDIUM_CELL_SIZE / BLOCK_SIZE; + break; + case SEGMENT_TYPE_BIG: + num = SEGMENT_TYPE_BIG_CELL_SIZE / BLOCK_SIZE; + break; + default: + break; + } + + blockCount = num; + +#ifndef YCSB_TEST + while (num--) + { + PrefetchBlockManagerEntry *entry = new PrefetchBlockManagerEntry(); + entry->blockID = firstBlockID + num; + entry->currentOffset = 0; + entry->isOccupied = PREFETCH_BLOCK_STATUS_IDLE; + prefetchBlocks.insert(std::pair(entry->blockID, *entry)); + } +#endif + +} + +PrefetchBlockManager::~PrefetchBlockManager() +{ + prefetchBlocks.clear(); +} + +uint64_t +PrefetchBlockManager::getNextBlock() +{ + uint32_t count = blockCount - currentID; + if (count <= 0) + { + return 0; + } + currentID++; + return currentID + firstBlockID - 1; +} + +uint64_t +PrefetchBlockManager::blockAllocate(uint64_t rowSize) +{ + if (blockCount == 0) + { + return 0; + } + // struct timespec time; + // clock_gettime(CLOCK_REALTIME, &time); + srand((unsigned)time(NULL)); + std::map::iterator iter; + uint32_t count = blockCount - currentID; + + while (count--) + { + uint64_t blockID = currentID + firstBlockID; + iter = prefetchBlocks.find(blockID); + + if (!iter->second.isOccupied) + { + if (rowSize < (BLOCK_SIZE - iter->second.currentOffset)) + { + iter->second.isOccupied = PREFETCH_BLOCK_STATUS_BUSY; + return blockID; + } + } + + currentID++; + } + + return 0; +} + +uint32_t +PrefetchBlockManager::setBlockState(uint64_t blockID, bool isOccupied) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + iter->second.isOccupied = isOccupied; + return SUCCESS; + } + else + { + return PREFETCH_BLOCK_NOT_FOUND; + } +} + +uint32_t +PrefetchBlockManager::setOffset(uint64_t blockID, uint64_t currentOffset) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + iter->second.currentOffset = currentOffset; + return SUCCESS; + } + else + { + return PREFETCH_BLOCK_NOT_FOUND; + } +} + +uint32_t +PrefetchBlockManager::getOffset(uint64_t blockID, uint64_t ¤tOffset) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + currentOffset = iter->second.currentOffset; + return SUCCESS; + } + else + { + return PREFETCH_BLOCK_NOT_FOUND; + } +} + +uint32_t +PrefetchBlockManager::advanceOffset(uint64_t blockID, uint64_t rowSize) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + iter->second.currentOffset += rowSize; + return SUCCESS; + } + else + { + return PREFETCH_BLOCK_NOT_FOUND; + } +} + +uint32_t +PrefetchBlockManager::isOccupied(uint64_t blockID, bool &status) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + status = iter->second.isOccupied; + return SUCCESS; + } + else + { + return PREFETCH_BLOCK_NOT_FOUND; + } +} + +uint32_t +PrefetchBlockManager::getPrefetchBlockEntry(uint64_t blockID, + PrefetchBlockManagerEntry &entry) +{ + entry = prefetchBlocks[blockID]; + return SUCCESS; +} + +uint32_t +PrefetchBlockManager::getAllPrefetchBlockEntrys(std::map &entrys) +{ + entrys = prefetchBlocks; + return SUCCESS; +} + +uint32_t +PrefetchBlockManager::lockBlock(uint64_t blockID) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + if (iter != prefetchBlocks.end()) + { + if (iter->second.isOccupied == PREFETCH_BLOCK_STATUS_BUSY) + { + return BLOCK_OCCUPIED; + } + iter->second.isOccupied = PREFETCH_BLOCK_STATUS_BUSY; + return SUCCESS; + } + return PREFETCH_BLOCK_NOT_FOUND; +} + +uint32_t +PrefetchBlockManager::unlockBlock(uint64_t blockID) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + if (iter != prefetchBlocks.end()) + { + if (iter->second.isOccupied == PREFETCH_BLOCK_STATUS_BUSY) + { + iter->second.isOccupied = PREFETCH_BLOCK_STATUS_IDLE; + } + return SUCCESS; + } + return PREFETCH_BLOCK_NOT_FOUND; +} + +bool +PrefetchBlockManager::isCached(uint64_t blockID, uint64_t &pageID) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end() && iter->second.isCached) + { + pageID = iter->second.pageID; + return true; + } + return false; +} + +uint32_t +PrefetchBlockManager::setIsCached(uint64_t blockID, bool isCached, uint64_t pageID) +{ + std::map::iterator iter; + iter = prefetchBlocks.find(blockID); + + if (iter != prefetchBlocks.end()) + { + iter->second.isCached = isCached; + if (isCached) + { + iter->second.pageID = pageID; + } + return SUCCESS; + } + return PREFETCH_BLOCK_NOT_FOUND; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RootTable.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RootTable.cc new file mode 100644 index 000000000..72f05ef87 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RootTable.cc @@ -0,0 +1,244 @@ +/** + * @file RootTable.cc + * @brief RootTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "RootTable.h" + +namespace LightTable { + +RootTable::RootTable(bool usePrimaryKeyIndex) + : SystemTable(ROOTTABLE_TABLE_ID, "RootTable", ROOTTABLE_FIRST_BLOCK_ID, + ROOTTABLE_SEGMENT_ID, SEGMENT_TYPE_SMALL, + usePrimaryKeyIndex) { + Schema::SchemaEntry entry1 = {1, "tableID", "uint64_t", 8, true, false}; + schema.appendEntry(entry1); + Schema::SchemaEntry entry2 = {2, "tableName", "string", TABLE_NAME_LENGTH, + false, false}; + schema.appendEntry(entry2); + Schema::SchemaEntry entry3 = {3, "firstBlockID", "uint64_t", 8, false, false}; + schema.appendEntry(entry3); + Schema::SchemaEntry entry4 = {4, "segmentID", "uint64_t", 8, false, false}; + schema.appendEntry(entry4); + Schema::SchemaEntry entry5 = {5, "segmentType", "uint32_t", 4, false, false}; + schema.appendEntry(entry5); + Schema::SchemaEntry entry6 = {6, "user", "string", USER_NAME_LENGTH, + false, true}; + schema.appendEntry(entry6); + Schema::SchemaEntry entry7 = {7, "time", "time", 60, false, false}; + schema.appendEntry(entry7); +} + +uint32_t RootTable::tableNameToTableID(const char *tableName, + uint64_t &tableID) { + std::map::iterator iter = tableTupleMap.begin(); + + while (iter != tableTupleMap.end()) { + if (0 == strncmp(tableName, iter->second.tableName, TABLE_NAME_LENGTH)) { + tableID = iter->second.tableID; + return SUCCESS; + } + iter++; + } + return TABLE_NOT_FOUND; +} + +uint32_t RootTable::deleteTableInfo(uint64_t tableID) { + uint32_t ret = SUCCESS; + + RowLocation rowLocation; + uint32_t bucketID = hashTable.hashToBucket(tableID); + ret = hashTable.getRowLocation(bucketID, tableID, rowLocation); + + uint64_t deleteBlockID = rowLocation.blockID; + uint64_t deleteRowOffset = rowLocation.rowOffset; + char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH]; + uint32_t size = 0; + + while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS) + ; + uint64_t currentOffset = 0; + ret = prefetchBlockManager.getOffset(rowLocation.blockID, currentOffset); + if (ret != SUCCESS) { + return ret; + } + + if (rowLocation.rowOffset + schema.getEntrySize() != currentOffset) { + // RowBufferItem::Tuple item; + // uint8_t str[ROOTTABLE_TUPLE_SIZE]; + // memcpy(str, &tableTupleMap[tableID], sizeof(tableTupleMap[tableID])); + // item.blockID = rowLocation.blockID; + // item.rowOffset = rowLocation.rowOffset; + // item.isDelete = true; + // RowBufferItem *rowBufferItem = new RowBufferItem(item); + // buf->append(rowBufferItem); + + std::map locationItems; + hashTable.refreshRowLocations(deleteBlockID, deleteRowOffset, + schema.getEntrySize(), locationItems); + // std::map::iterator iter = locationItems.begin(); + // while (iter != locationItems.end()) { + // MetaHandle::serialize(std::pair(iter->first, + // iter->second), tableID, hashTableMetaData, size); + // MetaBufferItem *nextMeta= new MetaBufferItem(tableID, hashTableMetaData, + // size); buf->append(nextMeta); + // } + } + + hashTable.deleteItem(bucketID, tableID); + rowLocation.blockID = 0; + // MetaHandle::serialize(std::pair(tableID, + // rowLocation), + // ROOTTABLE_TABLE_ID, hashTableMetaData, size); + // MetaBufferItem *hashTableMeta= new MetaBufferItem(ROOTTABLE_TABLE_ID, + // hashTableMetaData, size); + // buf->append(hashTableMeta); + + uint64_t newOffset = currentOffset - schema.getEntrySize(); + prefetchBlockManager.setOffset(deleteBlockID, newOffset); + + PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry; + prefetchBlockManager.unlockBlock(deleteBlockID); + prefetchBlockManager.getPrefetchBlockEntry(deleteBlockID, blockEntry); + // char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH]; + // MetaHandle::serialize(blockEntry, ROOTTABLE_TABLE_ID, pbmData, size); + // MetaBufferItem *pbmMeta = new MetaBufferItem(ROOTTABLE_TABLE_ID, pbmData, + // size); buf->append(pbmMeta); + + if (tableTupleMap.erase(tableID) == 1) { + return SUCCESS; + } + return ERROR_TABLE; +} + +uint32_t RootTable::appendTableInfo(TableTuple tableTuple) { + uint32_t err = SUCCESS; + + tableTupleMap.insert( + std::pair(tableTuple.tableID, tableTuple)); + + uint64_t blockID = 0; + uint64_t blockOffset = 0; + blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize()); + err = prefetchBlockManager.getOffset(blockID, blockOffset); + + if (err != SUCCESS) { + // TODO allocate new seg + return GET_AVAILABLE_BLOCK_ERROR; + } + + // RowBufferItem::Tuple item; + // uint8_t str[ROOTTABLE_TUPLE_SIZE]; + // memcpy(str, &tableTuple, sizeof(tableTuple)); + // item.row = str; + // item.blockID = blockID; + // item.rowOffset = blockOffset; + // item.isDelete = false; + // RowBufferItem *rowBufferItem = new RowBufferItem(item); + // buf->append(rowBufferItem); + + uint32_t bucketID = hashTable.hashToBucket(tableTuple.tableID); + RowLocation rowLocation = {blockID, blockOffset}; + err = hashTable.addItem(bucketID, tableTuple.tableID, rowLocation); + + if (err != SUCCESS) { + return err; + } + + // uint32_t size = 0; + // char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH]; + // MetaHandle::serialize(std::pair(tableTuple.tableID, + // rowLocation), tableID, hashTableMetaData, size); + // MetaBufferItem *hashTableMeta = new MetaBufferItem(tableID, + // hashTableMetaData, + // size); + // buf->append(hashTableMeta); + + err = prefetchBlockManager.advanceOffset(blockID, schema.getEntrySize()); + prefetchBlockManager.setBlockState(blockID, PREFETCH_BLOCK_STATUS_IDLE); + + // PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry; + // prefetchBlockManager.getPrefetchBlockEntry(blockID, blockEntry); + // char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH]; + // MetaHandle::serialize(blockEntry, tableID, pbmData, size); + // MetaBufferItem *pbmMeta = new MetaBufferItem(tableID, pbmData, size); + // buf->append(pbmMeta); + + return SUCCESS; +} + +uint32_t RootTable::getTableInfo(uint64_t tableID, + TableTuple &tableTupleResult) { + std::map::iterator iter; + iter = tableTupleMap.find(tableID); + + if (iter != tableTupleMap.end()) { + tableTupleResult = iter->second; + return SUCCESS; + } + tableTupleResult.tableID = 0; + return TABLE_TUPLE_NOT_FOUND; +} + +uint32_t RootTable::modifyTableInfo(uint64_t tableID, TableTuple tableTuple) { + uint32_t ret = SUCCESS; + std::map::iterator iter; + iter = tableTupleMap.find(tableID); + + if (iter != tableTupleMap.end()) { + iter->second = tableTuple; + + uint32_t bucketID = hashTable.hashToBucket(tableID); + RowLocation rowLocation; + ret = hashTable.getRowLocation(bucketID, tableID, rowLocation); + if (ret != SUCCESS) { + return ret; + } + // RowBufferItem::Tuple item; + // uint8_t str[ROOTTABLE_TUPLE_SIZE]; + // memcpy(str, &tableTuple, sizeof(tableTuple)); + // item.row = str; + // item.blockID = rowLocation.blockID; + // item.rowOffset = rowLocation.rowOffset; + // item.isDelete = false; + // RowBufferItem *rowBufferItem = new RowBufferItem(item); + // buf->append(rowBufferItem); + + return SUCCESS; + } + return TABLE_TUPLE_NOT_FOUND; +} + +uint32_t RootTable::getTableTuple(uint64_t tableID, TableTuple &tableTuple) { + std::map::iterator iter; + iter = tableTupleMap.find(tableID); + if (iter != tableTupleMap.end()) { + tableTuple = iter->second; + return SUCCESS; + } + return TABLE_TUPLE_NOT_FOUND; +} + +uint32_t RootTable::serializeRow(const TableTuple tableTuple, uint8_t *buf) { + memcpy(buf, &tableTuple, ROOTTABLE_TUPLE_SIZE); + + return SUCCESS; +} + +uint32_t RootTable::deserializeRow(const uint8_t *buf, + TableTuple *tableTupleResult) { + if (buf != NULL) { + memcpy(tableTupleResult, buf, ROOTTABLE_TUPLE_SIZE); + if (tableTupleResult->tableID != 0) { + return SUCCESS; + } + } + return BUFFER_EMPTY; +} + +uint64_t RootTable::getTableCount() { return tableTupleMap.size(); } + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RowMap.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RowMap.cc new file mode 100644 index 000000000..459b563ae --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/RowMap.cc @@ -0,0 +1,77 @@ +/** + * @file RowMap.cc + * @brief RowMap + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "RowMap.h" + +namespace LightTable { + +RowMap::RowMap() {} + +uint32_t RowMap::appendEntry(uint64_t key, uint64_t blockID, + uint64_t rowOffset) { + std::pair::iterator, bool> ret; + RowMapEntry rowMapEntry = {key, blockID, rowOffset}; + + ret = rowMapEntrys.insert(std::pair(key, rowMapEntry)); + + if (ret.second == false) { + return ADD_ROW_MAP_ENTRY_ERROR; + } + + return SUCCESS; +} + +uint32_t RowMap::deleteEntry(uint64_t key) { + if (rowMapEntrys.erase(key) == 1) { + return SUCCESS; + } + return ROW_MAP_ENTRY_NOT_FOUND; +} + +uint32_t RowMap::alterEntry(uint64_t key, uint64_t blockID, + uint64_t rowOffset) { + std::map::iterator iter; + iter = rowMapEntrys.find(key); + + if (iter != rowMapEntrys.end()) { + (iter->second).key = key; + (iter->second).blockID = blockID; + (iter->second).rowOffset = rowOffset; + return SUCCESS; + } + + return ROW_MAP_ENTRY_NOT_FOUND; +} + +uint32_t RowMap::queryEntry(uint64_t key, RowMapEntry &rowMapEntry) { + std::map::iterator iter; + iter = rowMapEntrys.find(key); + + if (iter != rowMapEntrys.end()) { + rowMapEntry = iter->second; + return SUCCESS; + } + + rowMapEntry.key = key; + rowMapEntry.blockID = 0; + rowMapEntry.rowOffset = 0; + return ROW_MAP_ENTRY_NOT_FOUND; +} + +uint32_t RowMap::queryAllEntry( + std::map &allRowMapEntrys) { + if (rowMapEntrys.empty() == false) { + allRowMapEntrys = rowMapEntrys; + return SUCCESS; + } + return ROW_MAP_ENTRY_NOT_FOUND; +} + +uint64_t RowMap::getRowCount() { return rowMapEntrys.size(); } + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Schema.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Schema.cc new file mode 100644 index 000000000..f139a3b3e --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Schema.cc @@ -0,0 +1,166 @@ +/** + * @file Schema.cc + * @brief table schema + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "Schema.h" + +namespace LightTable { + +Schema::Schema() { + primaryKeySchema.columnID = 1; + primaryKeySchema.isPrimaryKey = true; + columnCount = 0; + entrySize = 0; +} + +Schema::Schema(std::vector schemaEntrys) { + columnCount = 0; + entrySize = 0; + primaryKeySchema.columnID = 1; + primaryKeySchema.isPrimaryKey = true; + std::vector::iterator iter = schemaEntrys.begin(); + while (iter != schemaEntrys.end()) { + appendEntry(*iter); + iter++; + } +} + +uint32_t Schema::appendEntry(SchemaEntry &schemaEntry) { + if (schemaEntry.columnID != columnCount + 1) { + return COLUMN_ID_NOT_CONTINUOUS; + } + std::pair::iterator, bool> ret; + ret = schemaEntrys.insert(std::pair( + schemaEntry.columnID, schemaEntry)); + + if (ret.second == false) { + return ADD_SCHEMA_ENTRY_ERROR; + } + + if (schemaEntry.isPrimaryKey) { + primaryKeySchema = schemaEntry; + } + + columnCount++; + entrySize = entrySize + schemaEntry.length; + return SUCCESS; +} + +uint32_t Schema::deleteEntry(uint64_t columnID) { + if (columnID == primaryKeySchema.columnID) { + return DELETE_PARIMARY_KEY_COLUMN; + } + + uint64_t newColumnID = columnID; + SchemaEntry schemaEntry; + + std::map::iterator iter; + iter = schemaEntrys.find(columnID); + if (iter != schemaEntrys.end()) { + schemaEntry = iter->second; + schemaEntrys.erase(iter); + entrySize = entrySize - schemaEntry.length; + columnCount--; + + if (columnCount > 0) { + iter = schemaEntrys.find(newColumnID + 1); + + while (iter != schemaEntrys.end()) { + schemaEntry = iter->second; + schemaEntry.columnID = newColumnID; + + schemaEntrys.insert( + std::pair(newColumnID, schemaEntry)); + + schemaEntrys.erase(iter); + newColumnID++; + iter = schemaEntrys.find(newColumnID + 1); + } + } + return SUCCESS; + } + return SCHEMA_ENTRY_NOT_FOUND; +} + +uint32_t Schema::alterEntry(uint64_t columnID, SchemaEntry schemaEntry) { + if (schemaEntry.columnID != columnID) { + return COLUMN_ID_NOT_CONTINUOUS; + } + + std::map::iterator iter; + iter = schemaEntrys.find(columnID); + + entrySize = entrySize - (iter->second).length; + + if (iter != schemaEntrys.end()) { + iter->second = schemaEntry; + primaryKeySchema = schemaEntry; + entrySize = entrySize + (iter->second).length; + return SUCCESS; + } + return SCHEMA_ENTRY_NOT_FOUND; +} + +uint32_t Schema::getColumnID(const char *columnName, uint64_t &columnID) { + std::map::iterator iter = schemaEntrys.begin(); + while (iter != schemaEntrys.end()) { + if (strncmp(iter->second.columnName, columnName, iter->second.length) == + 0) { + columnID = (iter->second).columnID; + return SUCCESS; + } + iter++; + } + return COLUMN_ITEM_NOT_FOUND; +} + +uint32_t Schema::queryEntry(uint64_t columnID, + Schema::SchemaEntry &schemaEntry) { + std::map::iterator iter; + iter = schemaEntrys.find(columnID); + + if (iter != schemaEntrys.end()) { + schemaEntry = iter->second; + return SUCCESS; + } + + schemaEntry.columnID = 0; + return SCHEMA_ENTRY_NOT_FOUND; +} + +uint32_t Schema::queryAllEntry( + std::map &allSchemaEntrys) { + allSchemaEntrys = schemaEntrys; + if (schemaEntrys.empty() == false) { + return SUCCESS; + } + return SCHEMA_ENTRY_NOT_FOUND; +} + +uint64_t Schema::getEntrySize() { return entrySize; } + +void Schema::setEntrySize(uint64_t entrySize) { this->entrySize = entrySize; } + +uint64_t Schema::getColumnCount() { return columnCount; } + +void Schema::setColumnCount(uint64_t columnCount) { + this->columnCount = columnCount; +} + +uint32_t Schema::getPrimaryKeyLength() { return primaryKeySchema.length; } + +Schema::SchemaEntry &Schema::getPrimaryKeySchema() { return primaryKeySchema; } + +uint32_t Schema::drop() { + schemaEntrys.clear(); + primaryKeySchema.length = 0; + columnCount = 0; + entrySize = 0; + return SUCCESS; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SegmentManager.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SegmentManager.cc new file mode 100644 index 000000000..bf738ab98 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SegmentManager.cc @@ -0,0 +1,180 @@ +/** + * @file SegmentManager.cc + * @brief SegmentManager + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "SegmentManager.h" + +namespace LightTable { + +SegmentManager::SegmentManager() { + int blockID, num = 0; + + smallBitmap = new uint8_t[SEGMENT_TYPE_SMALL_BITMAP_SIZE]; + mediumBitmap = new uint8_t[SEGMENT_TYPE_MEDIUM_BITMAP_SIZE]; + bigBitmap = new uint8_t[SEGMENT_TYPE_BIG_BITMAP_SIZE]; + +#ifdef YCSB_TEST + blockID = SEGMENT_TYPE_BIG_START; + while (num < SEGMENT_TYPE_BIG_BITMAP_SIZE) { + Driver::read(bigBitmap + num, blockID); + num += BLOCK_SIZE; + blockID++; + } + +#else + num = 0; + blockID = SEGMENT_TYPE_SMALL_START; + while (num < SEGMENT_TYPE_SMALL_BITMAP_SIZE) { + Driver::read(smallBitmap + num, blockID); + num += BLOCK_SIZE; + blockID++; + } + + num = 0; + blockID = SEGMENT_TYPE_MEDIUM_START; + while (num < SEGMENT_TYPE_MEDIUM_BITMAP_SIZE) { + Driver::read(mediumBitmap + num, blockID); + num += BLOCK_SIZE; + blockID++; + } + + num = 0; + blockID = SEGMENT_TYPE_BIG_START; + while (num < SEGMENT_TYPE_BIG_BITMAP_SIZE) { + Driver::read(mediumBitmap + num, blockID); + num += BLOCK_SIZE; + blockID++; + } +#endif +} + +SegmentManager::~SegmentManager() { + delete[] smallBitmap; + delete[] mediumBitmap; + delete[] bigBitmap; +} + +uint32_t SegmentManager::setBitmap(SegmentType segmentType, uint64_t segmentID, + SegmentStatus segmentStatus) { + switch (segmentType) { + case SEGMENT_TYPE_SMALL: + return setBit(segmentID, smallBitmap, SEGMENT_TYPE_SMALL_START, + segmentStatus); + + case SEGMENT_TYPE_MEDIUM: + return setBit(segmentID, mediumBitmap, SEGMENT_TYPE_MEDIUM_START, + segmentStatus); + + case SEGMENT_TYPE_BIG: + return setBit(segmentID, bigBitmap, SEGMENT_TYPE_BIG_START, + segmentStatus); + + default: + return SEGMENT_TYPE_ERROR; + } +} + +uint32_t SegmentManager::getIdleSegment(SegmentType segmentType, + uint64_t &segmentID, + uint64_t &firstBlockID) { + uint32_t stateCode; + + switch (segmentType) { + case SEGMENT_TYPE_SMALL: + stateCode = + findFirstIdle(smallBitmap, SEGMENT_TYPE_SMALL_BITMAP_SIZE, segmentID); + + if (stateCode != SUCCESS) return stateCode; + + firstBlockID = SEGMENT_TYPE_SMALL_START + + (SEGMENT_TYPE_SMALL_CELL_SIZE / BLOCK_SIZE) * segmentID; + + return setBit(segmentID, smallBitmap, SEGMENT_TYPE_SMALL_START, + SEGMENT_STATUS_BUSY); + + case SEGMENT_TYPE_MEDIUM: + stateCode = findFirstIdle(mediumBitmap, SEGMENT_TYPE_MEDIUM_BITMAP_SIZE, + segmentID); + + if (stateCode != SUCCESS) return stateCode; + + firstBlockID = SEGMENT_TYPE_MEDIUM_START + + (SEGMENT_TYPE_MEDIUM_CELL_SIZE / BLOCK_SIZE) * segmentID; + + return setBit(segmentID, mediumBitmap, SEGMENT_TYPE_MEDIUM_START, + SEGMENT_STATUS_BUSY); + + case SEGMENT_TYPE_BIG: + stateCode = + findFirstIdle(bigBitmap, SEGMENT_TYPE_BIG_BITMAP_SIZE, segmentID); + + if (stateCode != SUCCESS) return stateCode; + + firstBlockID = SEGMENT_TYPE_BIG_START + + (SEGMENT_TYPE_BIG_CELL_SIZE / BLOCK_SIZE) * segmentID; + + return setBit(segmentID, bigBitmap, SEGMENT_TYPE_BIG_START, + SEGMENT_STATUS_BUSY); + + default: + return SEGMENT_TYPE_ERROR; + } +} + +uint32_t SegmentManager::setBit(uint64_t segmentID, uint8_t *buf, + uint64_t segmentStart, + SegmentStatus segmentStatus) { + uint32_t integer, remainder; + uint8_t tmp = 1; + + integer = segmentID / 8; + remainder = segmentID % 8; + + tmp = tmp << remainder; + if (segmentStatus == SEGMENT_STATUS_BUSY) { + buf[integer] |= tmp; + } else if (segmentStatus == SEGMENT_STATUS_IDLE) { + buf[integer] &= ~tmp; + } else { + return SEGMENT_STATUS_ERROR; + } + + uint64_t num = integer / BLOCK_SIZE; + uint64_t blockID = num + segmentStart; + + Driver::write(buf + num * BLOCK_SIZE, blockID); + + return SUCCESS; +} + +uint32_t SegmentManager::findFirstIdle(uint8_t *buf, uint64_t segmentNum, + uint64_t &segmentID) { + uint64_t num, place; + uint8_t res, tmp, isfound; + + for (isfound = num = 0; num < segmentNum; num++) { + for (place = 0; place < 8; place++) { + tmp = 0x01 << place; + tmp = tmp & buf[num]; + res = (tmp >> place); + if (res == 0) { + isfound = 1; + break; + } + } + if (isfound == 1) break; + } + + if (isfound == 0) { + return SEGMENT_IS_EXHAUSTED; + } + segmentID = num * 8 + place; + + return SUCCESS; +} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SystemTable.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SystemTable.cc new file mode 100644 index 000000000..183035e0b --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/SystemTable.cc @@ -0,0 +1,31 @@ +/** + * @file SystemTable.cc + * @brief SystemTable + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "SystemTable.h" + +namespace LightTable { + +SystemTable::SystemTable(uint64_t tableID, const char *tableName, + uint64_t firstBlockID, uint64_t segmentID, + SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex) + : Table(tableID, tableName, firstBlockID, segmentID, segmentType, + schemaEntrys, usePrimaryKeyIndex) + +{} + +SystemTable::SystemTable(uint64_t tableID, const char *tableName, + uint64_t firstBlockID, uint64_t segmentID, + SegmentType segmentType, bool usePrimaryKeyIndex) + : Table(tableID, tableName, firstBlockID, segmentID, segmentType, + usePrimaryKeyIndex) + +{} + +} // namespace LightTable diff --git a/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Table.cc b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Table.cc new file mode 100644 index 000000000..abaa5fdd0 --- /dev/null +++ b/Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210/tablestorage/src/storage/Table.cc @@ -0,0 +1,639 @@ +/** + * @file Table.cc + * @brief Table + * @version 0.1 + * @author SYS Lab + * @date 2022.11.01 + */ + +#include "Table.h" + +namespace LightTable { + +Table::Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + std::vector schemaEntrys, + bool usePrimaryKeyIndex) + : prefetchBlockManager(firstBlockID, segmentID, segmentType), + schema(schemaEntrys), + rowMap(), + tupleNum(0), + t(time(0)), + indexLocation{TABLE1_INDEX_BLOCKID, META_ENTRY_NUM}, + metaLogLocation{TABLE1_META_LOG_BLOCKID, META_ENTRY_NUM}, + indexLogLocation{TABLE1_INDEX_LOG_BLOCKID, META_ENTRY_NUM}, + dataLogLocation{TABLE1_DATA_LOG_BLOCKID, META_ENTRY_NUM} { + this->tableID = tableID; + memcpy(this->tableName, tableName, sizeof(tableName) + 1); + this->usePrimaryKeyIndex = usePrimaryKeyIndex; + std::map allSchemaEntrys; + schema.queryAllEntry(allSchemaEntrys); +} + +Table::Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID, + uint64_t segmentID, SegmentType segmentType, + bool usePrimaryKeyIndex) + : prefetchBlockManager(firstBlockID, segmentID, segmentType), + schema(), + rowMap() { + this->tableID = tableID; + memcpy(this->tableName, tableName, sizeof(tableName) + 1); + // this->buf = buf; + // this->queryCache = queryCache; + // this->blockPreload = blockPreload; + this->usePrimaryKeyIndex = usePrimaryKeyIndex; +} + +uint32_t Table::drop() { + uint32_t ret = SUCCESS; + + ret = truncate(); + if (ret != SUCCESS) { + return ret; + } + + schema.drop(); + + // std::map allSchemaEntrys; + // schema.queryAllEntry(allSchemaEntrys); + // uint64_t columnCount = allSchemaEntrys.size(); + // uint64_t length = 10 + columnCount * (SCHEMA_ENTRY_SERIALIZED_LENGTH + 1); + // char *schemaData = new char[length]; + // uint32_t size = 0; + // MetaHandle::serialize(allSchemaEntrys, tableID, schemaData, size); + // MetaBufferItem *schemaMetaItem = new MetaBufferItem(tableID, schemaData, + // size); buf->append(schemaMetaItem); delete[] schemaData; + + return SUCCESS; +} + +// uint32_t Table::truncate() { +// uint32_t size = 0; +// char *pbmData; +// MetaBufferItem *pbmMeta; +// char *hashTableData; +// MetaBufferItem *hashTableMeta; +// std::map entrys; +// prefetchBlockManager.getAllPrefetchBlockEntrys(entrys); +// std::map::iterator +// pbmIter = entrys.begin(); +// while (pbmIter != entrys.end()) { +// prefetchBlockManager.lockBlock(pbmIter->first); +// prefetchBlockManager.setOffset(pbmIter->first, 0); +// prefetchBlockManager.unlockBlock(pbmIter->first); +// pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH]; +// MetaHandle::serialize(pbmIter->second, tableID, pbmData, size); +// pbmMeta = new MetaBufferItem(tableID, pbmData, size); +// // buf->append(pbmMeta); +// pbmIter++; +// } + +// std::vector buckets; +// hashTable.getAllBuckets(buckets); +// for (uint32_t i = 0; i < buckets.size(); i++) { +// std::map bucketItems; +// buckets[i]->getAllBucketItems(bucketItems); +// std::map::iterator itemIter; +// itemIter = bucketItems.begin(); +// while (itemIter != bucketItems.end()) { +// itemIter->second.blockID = 0; +// hashTableData = new char[ROWMAP_SERIALIZED_LENGTH]; +// MetaHandle::serialize( +// std::pair(itemIter->first, itemIter->second), +// tableID, hashTableData, size); +// hashTableMeta = new MetaBufferItem(tableID, hashTableData, size); +// // buf->append(hashTableMeta); +// itemIter = bucketItems.erase(itemIter); +// itemIter++; +// } +// } + +// return SUCCESS; +// } + +uint32_t Table::parsePrimaryKey(const uint8_t *rowData, char *primaryKeyData) { + if (strncmp(schema.getPrimaryKeySchema().type, "uint64_t", 8) == 0) { + uint64_t primaryKeyI64 = 0; + memcpy(&primaryKeyI64, rowData, schema.getPrimaryKeySchema().length); + sprintf(primaryKeyData, "%ld", primaryKeyI64); + } else if (strncmp(schema.getPrimaryKeySchema().type, "string", 6) == 0) { + memcpy(primaryKeyData, rowData, schema.getPrimaryKeySchema().length); + } else { + return TYPE_INVALID; + } + return SUCCESS; +} + +uint32_t Table::insertRow(const uint8_t *rowData) { + uint32_t ret = SUCCESS; + uint64_t blockID = 0; + uint64_t blockOffset = 0; + uint8_t blockData[BLOCK_SIZE]; + tupleNum++; + t = time(0); + +#ifndef YCSB_TEST + blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize()); + while (blockID == 0) { + blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize()); + } + ret = prefetchBlockManager.getOffset(blockID, blockOffset); + if (ret != SUCCESS) { + // TODO allocate new seg + return ret; + } +#else + blockID = prefetchBlockManager.getNextBlock(); +#endif + + if (blockOffset != 0) { + Driver::read(blockData, blockID); + } + + memcpy(blockData + blockOffset, rowData, schema.getEntrySize()); +#ifdef SDCARD_TEST + start_time = std::chrono::steady_clock::now(); +#endif + Driver::write(blockData, blockID); +#ifdef SDCARD_TEST + for (int i = 0; i < SDCARD_TEST_NUM; ++i) + Driver::write(blockData, SDCARD_TEST_WRITE_BLOCKID + 100 * i); + end_time = std::chrono::steady_clock::now(); + diff_sdcard_write = std::chrono::duration_cast( + end_time - start_time) + .count(); +#endif + + char *primaryKeyData = new char[schema.getPrimaryKeySchema().length]; + ret = parsePrimaryKey(rowData, primaryKeyData); + if (ret != SUCCESS) { + delete[] primaryKeyData; + prefetchBlockManager.unlockBlock(blockID); + return ret; + } + + uint64_t key = hashTable.getMapKey(primaryKeyData); + uint32_t bucketID = hashTable.hashToBucket(key); + RowLocation rowLocation = {blockID, blockOffset}; + ret = hashTable.addItem(bucketID, key, rowLocation); + + if (ret != SUCCESS) { + delete[] primaryKeyData; + prefetchBlockManager.unlockBlock(blockID); + return ret; + } + +#ifdef TABLE_STORAGE_SECOND_INDEX + ret = secondaryIndex.insert(primaryKeyData, + schema.getPrimaryKeySchema().length, rowLocation); + if (ret != SUCCESS) { + delete[] primaryKeyData; + prefetchBlockManager.unlockBlock(blockID); + return ret; + } +#endif + +#ifndef YCSB_TEST + ret = prefetchBlockManager.advanceOffset(blockID, schema.getEntrySize()); + prefetchBlockManager.setBlockState(blockID, PREFETCH_BLOCK_STATUS_IDLE); +#endif + + delete[] primaryKeyData; + + return ret; +} + +uint32_t Table::insertRow_logging(const uint8_t *rowData) { + uint32_t ret = SUCCESS; + uint64_t blockID = 0; + uint64_t blockOffset = 0; + + tupleNum++; + t = time(0); + + char *primaryKeyData = new char[schema.getPrimaryKeySchema().length]; + ret = parsePrimaryKey(rowData, primaryKeyData); + if (ret != SUCCESS) { + delete[] primaryKeyData; + prefetchBlockManager.unlockBlock(blockID); + return ret; + } + + uint64_t key = hashTable.getMapKey(primaryKeyData); + uint32_t bucketID = hashTable.hashToBucket(key); + RowLocation rowLocation = {blockID, blockOffset}; + + ret = hashTable.addItem(bucketID, key, rowLocation); + if (ret != SUCCESS) { + delete[] primaryKeyData; + prefetchBlockManager.unlockBlock(blockID); + return ret; + } + + metaLogEntry metaEntry{.tableID = tableID, .tupleNum = tupleNum, .t = t}; + + if (metaLogLocation.emptySlotNum != 0) { + memcpy(metaLog + (META_ENTRY_SIZE * + (META_ENTRY_NUM - metaLogLocation.emptySlotNum)), + &metaEntry, sizeof(metaLogEntry)); + } else { + metaLogLocation.blockID++; + metaLogLocation.emptySlotNum = META_ENTRY_NUM; + memset(metaLog, 0, BLOCK_SIZE); + memcpy(metaLog, &metaEntry, sizeof(metaLogEntry)); + } + Driver::write(metaLog, metaLogLocation.blockID); + metaLogLocation.emptySlotNum--; + + uint32_t size = 0; + char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH]; +// MetaHandle::serialize(std::pair(key, rowLocation), +// tableID, hashTableMetaData, size); +// MetaBufferItem *hashTableMeta = +// new MetaBufferItem(tableID, hashTableMetaData, size); +// if (indexLogLocation.emptySlotNum != 0) { +// memcpy(indexLog + (META_ENTRY_SIZE * +// (META_ENTRY_NUM - indexLogLocation.emptySlotNum)), +// hashTableMeta->metaItem, +// META_ENTRY_SIZE > hashTableMeta->metaItemSize +// ? hashTableMeta->metaItemSize +// : META_ENTRY_SIZE); +// } else { +// indexLogLocation.blockID++; +// indexLogLocation.emptySlotNum = META_ENTRY_NUM; +// memset(indexLog, 0, BLOCK_SIZE); +// memcpy(indexLog, hashTableMeta->metaItem, +// META_ENTRY_SIZE > hashTableMeta->metaItemSize +// ? hashTableMeta->metaItemSize +// : META_ENTRY_SIZE); +// } + Driver::write(indexLog, indexLogLocation.blockID); + indexLogLocation.emptySlotNum--; + + memcpy(metaData, &tupleNum, 8); + memcpy(metaData + 8, &t, sizeof(std::time_t)); + Driver::write(metaData, TABLE1_META_BLOCKID); + +// if (indexLocation.emptySlotNum != 0) { +// memcpy(indexData + (META_ENTRY_SIZE * +// (META_ENTRY_NUM - indexLocation.emptySlotNum)), +// hashTableMeta->metaItem, +// META_ENTRY_SIZE > hashTableMeta->metaItemSize +// ? hashTableMeta->metaItemSize +// : META_ENTRY_SIZE); +// } else { +// indexLocation.blockID++; +// indexLocation.emptySlotNum = META_ENTRY_NUM; +// memset(indexData, 0, BLOCK_SIZE); +// memcpy(indexData, hashTableMeta->metaItem, +// META_ENTRY_SIZE > hashTableMeta->metaItemSize +// ? hashTableMeta->metaItemSize +// : META_ENTRY_SIZE); +// } + Driver::write(indexData, indexLocation.blockID); + indexLocation.emptySlotNum--; + + uint8_t blockData[BLOCK_SIZE]; + blockID = dataLogLocation.blockID; + memcpy(blockData, rowData, schema.getEntrySize()); + Driver::write(blockData, blockID); + dataLogLocation.blockID++; + + blockID = prefetchBlockManager.getNextBlock(); + if (blockOffset != 0) { + Driver::read(blockData, blockID); + } + memcpy(blockData + blockOffset, rowData, schema.getEntrySize()); + Driver::write(blockData, blockID); + + delete[] hashTableMetaData; +// delete hashTableMeta; + delete[] primaryKeyData; + + return ret; +} + +uint32_t Table::deleteRow(const char *primaryKey) { + uint32_t ret = SUCCESS; + + RowLocation rowLocation; + ret = locateRow(primaryKey, rowLocation); + if (ret != SUCCESS) { + return ret; + } + uint64_t deleteBlockID = rowLocation.blockID; + uint64_t deleteRowOffset = rowLocation.rowOffset; + + while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS) + ; + uint64_t currentOffset = 0; + ret = prefetchBlockManager.getOffset(rowLocation.blockID, currentOffset); + uint64_t key = hashTable.getMapKey(primaryKey); + + if (rowLocation.rowOffset + schema.getEntrySize() != currentOffset) { + ret = deleteRowData(key, rowLocation); + if (ret != SUCCESS) { + return ret; + } + } + + uint32_t size = 0; + uint32_t bucketID = hashTable.hashToBucket(key); + hashTable.deleteItem(bucketID, key); + char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH]; + rowLocation.blockID = 0; + // MetaHandle::serialize(std::pair(key, rowLocation), + // tableID, hashTableMetaData, size); + // MetaBufferItem *hashTableMeta= new MetaBufferItem(tableID, + // hashTableMetaData, + // size); + // buf->append(hashTableMeta); + + if (deleteRowOffset + schema.getEntrySize() != currentOffset) { + std::map locationItems; + hashTable.refreshRowLocations(deleteBlockID, deleteRowOffset, + schema.getEntrySize(), locationItems); + // std::map::iterator iter = locationItems.begin(); + // while (iter != locationItems.end()) { + // MetaHandle::serialize(std::pair(iter->first, + // iter->second), tableID, hashTableMetaData, + // size); + // MetaBufferItem *nextMeta= new MetaBufferItem(tableID, + // hashTableMetaData, + // size); + // buf->append(nextMeta); + // } + } + + uint64_t newOffset = currentOffset - schema.getEntrySize(); + prefetchBlockManager.setOffset(deleteBlockID, newOffset); + prefetchBlockManager.unlockBlock(deleteBlockID); + PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry; + prefetchBlockManager.getPrefetchBlockEntry(deleteBlockID, blockEntry); + // char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH]; + // MetaHandle::serialize(blockEntry, tableID, pbmData, size); + // MetaBufferItem *pbmMeta = new MetaBufferItem(tableID, pbmData, size); + // buf->append(pbmMeta); + + return SUCCESS; +} + +uint32_t Table::deleteRowData(uint64_t key, RowLocation rowLocation) { + uint8_t blockData[BLOCK_SIZE]; + Driver::read(blockData, rowLocation.blockID); + + uint64_t start = rowLocation.rowOffset; + uint64_t nextRow = start + schema.getEntrySize(); + memcpy(blockData + start, blockData + nextRow, BLOCK_SIZE - nextRow); + + Driver::write(blockData, rowLocation.blockID); + + return SUCCESS; +} + +uint32_t Table::updateRow(const char *primaryKey, const uint8_t *rowData) { + uint32_t ret = SUCCESS; + + RowLocation rowLocation; + ret = locateRow(primaryKey, rowLocation); + if (ret == BUCKET_ITEM_NOT_FOUND) { + return insertRow(rowData); + } else if (ret != SUCCESS) { + return ret; + } + + uint8_t blockData[BLOCK_SIZE]; + + while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS) + ; + Driver::read(blockData, rowLocation.blockID); + + // char *undoTuple = new char[schema.getEntrySize()]; + // memcpy(undoTuple, blockData+rowLocation.rowOffset, schema.getEntrySize()); + // UndoLogEntry *logEntry = new UndoLogEntry(this->tableID, primaryKey, + // schema.getPrimaryKeyLength(), + // undoTuple, schema.getEntrySize(), + // rowLocation.blockID, rowLocation.rowOffset); + // buf->append(logEntry); + + memcpy(blockData + rowLocation.rowOffset, rowData, schema.getEntrySize()); + + if (SUCCESS != Driver::write(blockData, rowLocation.blockID)) { + // memcpy(blockData + rowLocation.rowOffset, undoTuple, + // schema.getEntrySize()); + // ret = UPDATE_TUPLE_ERROR; + } + + // logEntry->expireUndoLog(); + prefetchBlockManager.unlockBlock(rowLocation.blockID); + return ret; +} + +uint32_t Table::locateRow(const char *primaryKey, RowLocation &rowLocation) { + uint32_t ret = SUCCESS; + + uint64_t key = hashTable.getMapKey(primaryKey); + uint32_t bucketID = hashTable.hashToBucket(key); + + ret = hashTable.getRowLocation(bucketID, key, rowLocation); + return ret; +} + +uint32_t Table::selectRow(const char *primaryKey, uint8_t *rowData) { + RowLocation rowLocation; + return selectRow(primaryKey, rowData, rowLocation); +} + +uint32_t Table::selectRow(const char *primaryKey, uint8_t *rowData, + RowLocation &rowLocation) { + uint32_t ret = SUCCESS; + uint8_t blockData[BLOCK_SIZE]; + + ret = locateRow(primaryKey, rowLocation); + if (ret != SUCCESS) { + return ret; + } + +#ifndef YCSB_TEST + while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS) + ; +#endif + +#ifdef SDCARD_TEST + start_time = std::chrono::steady_clock::now(); +#endif + Driver::read(blockData, rowLocation.blockID); +#ifdef SDCARD_TEST + for (int i = 0; i < SDCARD_TEST_NUM; ++i) + Driver::read(blockData, SDCARD_TEST_READ_BLOCKID + 100 * i); + end_time = std::chrono::steady_clock::now(); + diff_sdcard_read = (std::chrono::duration_cast( + end_time - start_time) + .count()); +#endif +#ifndef YCSB_TEST + prefetchBlockManager.unlockBlock(rowLocation.blockID); +#endif + + ret = getRowFromBlock(primaryKey, rowData, blockData, rowLocation); + + return ret; +} + +uint32_t Table::getRowFromBlock(const char *primaryKey, uint8_t *rowData, + const uint8_t *blockData, + const RowLocation rowLocation) { + uint32_t ret = SUCCESS; + + char *primaryKeyData = new char[schema.getPrimaryKeySchema().length]; + if (rowLocation.rowOffset + schema.getEntrySize() > BLOCK_SIZE) { + delete[] primaryKeyData; + return ROWLOCATION_INVALID; + } + + ret = parsePrimaryKey(blockData + rowLocation.rowOffset, primaryKeyData); + if (ret != SUCCESS) { + delete[] primaryKeyData; + return ret; + } + + if (strncmp(primaryKeyData, primaryKey, + schema.getPrimaryKeySchema().length) != 0) { + delete[] primaryKeyData; + return ERROR_TABLE; + } + + memcpy(rowData, blockData + rowLocation.rowOffset, schema.getEntrySize()); + + delete[] primaryKeyData; + return SUCCESS; +} + +uint32_t Table::selectColumnItem(const char *primaryKey, const char *columnName, + uint8_t *columnData) { + uint32_t ret = SUCCESS; + + uint8_t *rowData = new uint8_t[schema.getEntrySize()]; + RowLocation rowLocation; + ret = selectRow(primaryKey, rowData, rowLocation); + if (ret != SUCCESS) { + delete[] rowData; + return ret; + } + + uint64_t columnOffset = 0; + uint64_t columnID = 0; + uint32_t columnLength = 0; + ret = getColumnOffset(columnName, columnID, columnLength, columnOffset); + + memcpy(columnData, rowData + columnOffset, columnLength); + + delete[] rowData; + return SUCCESS; +} + +uint32_t Table::updateColumnItem(const char *primaryKey, const char *columnName, + const uint8_t *columnData) { + uint32_t ret = SUCCESS; + + uint8_t *blockData = new uint8_t[BLOCK_SIZE]; + RowLocation rowLocation; + ret = locateRow(primaryKey, rowLocation); + if (ret != SUCCESS) { + return ret; + } + +#ifndef YCSB_TEST + while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS) + ; +#endif + Driver::read(blockData, rowLocation.blockID); + + // char *undoTuple = new char[schema.getEntrySize()]; + // memcpy(undoTuple, blockData+rowLocation.rowOffset, schema.getEntrySize()); + // UndoLogEntry *logEntry = new UndoLogEntry(this->tableID, primaryKey, + // schema.getPrimaryKeyLength(), + // undoTuple, schema.getEntrySize(), + // rowLocation.blockID, rowLocation.rowOffset); + // buf->append(logEntry); + + uint64_t columnOffset, columnID; + uint32_t columnLength; + ret = getColumnOffset(columnName, columnID, columnLength, columnOffset); + + uint64_t columnStart = rowLocation.rowOffset + columnOffset; + memcpy(blockData + columnStart, columnData, columnLength); + + if (SUCCESS != Driver::write(blockData, rowLocation.blockID)) { + // memcpy(blockData + rowLocation.rowOffset, undoTuple, + // schema.getEntrySize()); + // ret = UPDATE_TUPLE_ERROR; + } + + // logEntry->expireUndoLog(); +#ifndef YCSB_TEST + prefetchBlockManager.unlockBlock(rowLocation.blockID); +#endif + + delete[] blockData; + return SUCCESS; +} + +uint32_t Table::updateColumnItem_logging(const char *primaryKey, + const char *columnName, + const uint8_t *columnData) { + // TODO + return SUCCESS; +} + +uint32_t Table::getColumnOffset(const char *columnName, uint64_t &columnID, + uint32_t &columnLength, uint64_t &offset) { + uint32_t ret = SUCCESS; + offset = 0; + columnID = 0; + columnLength = 0; + + ret = getColumnID(columnName, columnID); + if (ret != SUCCESS) { + return ret; + } + + std::map schemaEntrys; + ret = schema.queryAllEntry(schemaEntrys); + if (ret != SUCCESS) { + return ret; + } + + columnLength = schemaEntrys[columnID].length; + for (uint64_t i = 1; i < columnID; i++) { + offset += schemaEntrys[i].length; + } + + return SUCCESS; +} + +uint64_t Table::getRowCount() { return hashTable.getItemCount(); } + +uint64_t Table::getTableID() { return tableID; } + +char *Table::getTableName() { return tableName; } + +uint32_t Table::setTableID(uint64_t tableID) { + this->tableID = tableID; + return SUCCESS; +} + +uint32_t Table::setTableName(const char *tableName) { + memcpy(this->tableName, tableName, sizeof(tableName) + 1); + return SUCCESS; +} + +uint32_t Table::getColumnID(const char *columnName, uint64_t &columnID) { + return schema.getColumnID(columnName, columnID); +} + +Schema &Table::getSchema() { return schema; } + +HashTable &Table::getHashTable() { return hashTable; } + +} // namespace LightTable