add tablestorage for RT-Thread_Fusion_XiUOS

This commit is contained in:
yu 2022-11-30 15:27:07 +08:00
parent e5bf0cee41
commit 5a3954b726
50 changed files with 3931 additions and 0 deletions

View File

@ -0,0 +1,36 @@
menu "TableStorage"
config TABLE_STORAGE
bool "Using Table storage system"
default n
if TABLE_STORAGE
config TABLE_STORAGE_BASIC
bool "Enable TableStorage Basic"
default y
config TABLE_STORAGE_QUERY
bool "[System-level] Enable TableStorage Buffer"
default n
config TABLE_STORAGE_CACHE
bool "[System-level] Enable TableStorage Prefetcher"
default n
config TABLE_STORAGE_PREFETCH
bool "[Table-level] Enable TableStorage Query Cache"
default n
config TABLE_STORAGE_SECOND_INDEX
bool "[Table-level] Enable TableStorage Second Index"
default n
config TABLE_STORAGE_TRANSACTION
bool "(Other) Enable TableStorage Transaction"
default n
endif
endmenu

View File

@ -0,0 +1,153 @@
## README
TableStorage是一款面向泛在操作系统的轻量级表存储原型系统。TableStorage专注于泛在操作系统场景下结构化数据的存储与传统的多层堆叠式软件栈数据库 + 文件系统不同TableStorage避免过度分层从存储全栈角度进行跨层数据库的存储引擎 + 文件系统)的设计,主要包含以下三个属性
- 低冗余:去除文件抽象并直接将“表”存储到设备中, 避免功能冗余和不必要的软件开销
- 兼容性提供一组通用的API 以支持表级的存取操作,与传统数据库中的读写操作兼容
- 可集成:支持组件的深度集成,具体来说, 可以集成事务和执行引擎以满足复杂的事务和查询处理需求
### 开发板
- K210最小系统板Max bit
- SD卡配置
| 引脚 | 作用 | RW007板子 |
| :----------------: | :-------: | :-------: |
| io 27(印丝标注SCK) | SPI1_SCK | SCK |
| io 26(印丝标注SO) | SPI1_MISO | MISO |
| io 28(印丝标注SI) | SPI1_MOSI | MOSI |
| io 29 | CS/BOOT1 | CS |
### 编译说明
- 环境搭建
- 参考https://gitlink.org.cn/xuos/xiuos/tree/prepare_for_master/Ubiquitous%2FXiZi_IIoT%2Fboard%2Fkd233下的**开发环境搭建**小节搭建好XiUOS的开发环境
- 参考https://gitlink.org.cn/xuos/xiuos/tree/prepare_for_master/Ubiquitous%2FRT-Thread_Fusion_XiUOS%2Faiit_board%2Fk210搭建好XiUOS-RTThread的开发环境
- 配置XiUOS-RTThread基本环境
- SD卡的配置按照上表SD卡引脚说明配置
<img src="pic/image-20221123215723700.png" alt="image-20221123215723700" style="zoom: 33%;" />
- 其他推荐配置
<img src="pic/image-20221123215509060.png" alt="image-20221123215509060" style="zoom: 50%;" />
- (可选)如果编译时,出现定时器错误,则可以选择使用软件定时器
<img src="pic/image-20221123215939427.png" alt="image-20221123215939427" style="zoom: 33%;" />
- 配置TableStorage
```shell
scons --menuconfig
```
- 若不开启TableStorage组件则自动在SD卡上使用FATFS
- 若开启TableStorage组件则在SD卡上使用TableStorage默认打开了Enable TableStorage Basic模块当前版本仅支持Basic模块
<img src="pic/image-20221123213344406.png" alt="image-20221123213344406" style="zoom: 33%;" />
- 执行 scons 编译若编译正确无误在当前文件夹下生成rtthread.elf、rtthread.bin。其中rtthread.bin需要烧写到设备中进行运行
- 烧录及运行结果图
```shell
sudo kflash -t rtthread.bin -p /dev/ttyUSB0
```
- 烧录并运行无误,则
<img src="pic/image-20221123215143184.png" alt="image-20221123215143184" style="zoom: 33%;" />
### 调试
- 修改k210/rtconfig.py中的BUILD选项来配置debug模式并重新编译
- 安装openocd下载ubuntu版本64位
- 下载地址:[Releases · kendryte/openocd-kendryte (github.com)](https://github.com/kendryte/openocd-kendryte/releases),推荐下载地址为http://101.36.126.201:8011/kendryte-openocd-0.2.3-ubuntu64.tar.gz
- 安装
```shell
sudo apt install libusb-dev libftdi-dev libhidapi-dev
sudo mv kendryte-openocd-0.2.3-ubuntu64.tar.gz /opt
cd /opt
sudo tar -zxvf kendryte-openocd-0.2.3-ubuntu64.tar.gz
```
- 修改配置文件
sudo vim /opt/kendryte-openocd/tcl/k210.cfg并复制以下内容
```shell
# SiPEED USB-JTAG/TTL
interface ftdi
ftdi_device_desc "Dual RS232"
ftdi_vid_pid 0x0403 0x6010
ftdi_layout_init 0x0508 0x0f1b
ftdi_layout_signal nTRST -data 0x0200 -noe 0x0100
ftdi_layout_signal nSRST -data 0x0800 -noe 0x0400
jtag_rclk 3000
# server port
gdb_port 9999
telnet_port 4444
# add cpu target
set _CHIPNAME riscv
jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x04e4796b
set _TARGETNAME $_CHIPNAME.cpu
target create $_TARGETNAME riscv -chain-position $_TARGETNAME
# command
init
if {[ info exists pulse_srst]} {
ftdi_set_signal nSRST 0
ftdi_set_signal nSRST 1
ftdi_set_signal nSRST z
}
halt
```
- 调试器和Max bit开发板的硬件连线
<img src="pic/image-20221123220234259.png" alt="image-20221123220234259" style="zoom: 33%;" />
- 启动调试器
```shell
sudo /opt/kendryte-openocd/bin/openocd -f /opt/kendryte-openocd/tcl/k210.cfg
```
<img src="pic/image-20221123221828637.png" alt="image-20221123221828637" style="zoom: 33%;" />
- 在Ubiquitous/RT-Thread_Fusion_XiUOS/aiit_board/k210目录下连接调试器
```shell
/opt/xpack-riscv-none-embed-gcc-10.2.0-1.2/bin/riscv-none-embed-gdb rtthread.elf --eval-command="target remote 127.0.0.1:9999"
```
- gdb终端调试
<img src="pic/image-20221123222125802.png" alt="image-20221123222125802" style="zoom: 33%;" />
<img src="pic/image-20221123223127817.png" alt="image-20221123223127817" style="zoom:33%;" />
- vscode调试
<img src="pic/image-20221123225705614.png" alt="image-20221123225705614" style="zoom:33%;" />

View File

@ -0,0 +1,26 @@
from building import *
# get current directory
cwd = GetCurrentDir()
# the set of source files associated with this SConscript file.
src = Glob('src/common/*.cc')
src += Glob('src/execution/*.cc')
src += Glob('src/storage/*.cc')
# compile optional modules
if GetDepend(['TABLE_STORAGE_CACHE']):
src += ['src/modules/prefetcher.cc']
if GetDepend(['TABLE_STORAGE_CACHE']):
src += ['src/modules/queryCache.cc']
if GetDepend(['TABLE_STORAGE_CACHE']):
src += ['src/modules/buffer.cc']
if GetDepend(['TABLE_STORAGE_CACHE']):
src += ['src/modules/secondIndex.cc']
# include path
path = [cwd + '/include']
group = DefineGroup('TableStorage', src, depend = ['TABLE_STORAGE'], CPPPATH = path)
Return('group')

View File

@ -0,0 +1,35 @@
/**
* @file BufferItem.h
* @brief BufferItem
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef BUFFERITEM_H
#define BUFFERITEM_H
#include <stdint.h>
#include "Common.h"
namespace LightTable {
class BufferItem {
public:
BufferItem(uint64_t tableID);
virtual ~BufferItem();
// virtual void flush() const = 0;
uint64_t getTableID() { return tableID; }
protected:
uint64_t tableID;
DISALLOW_COPY_AND_ASSIGN(BufferItem);
};
} // namespace LightTable
#endif // BUFFERITEM_H

View File

@ -0,0 +1,144 @@
/**
* @file Common.h
* @brief parameter configuration
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef COMMON_H
#define COMMON_H
#include <rtconfig.h>
#include <time.h>
#include <chrono>
#include <queue>
namespace LightTable {
#define YCSB_TEST
extern int cacheReadNum;
extern int cacheWriteNum;
extern std::chrono::steady_clock::time_point start_time;
extern std::chrono::steady_clock::time_point end_time;
extern double diff_sdcard_read;
extern double diff_sdcard_write;
#define SYSTEM_MAX_CORES 0
#define SYSTEM_MAX_IO_QUEUES 8
#ifndef YCSB_TEST
#define NETWORK_MESSAGE_DATA_MAX_SIZE 4088
#define NETWORK_TRANSMIT_MAX_SIZE 4096
#else
#define NETWORK_MESSAGE_DATA_MAX_SIZE 51192
#define NETWORK_TRANSMIT_MAX_SIZE 51200
#endif
#define SDCARD_TEST_WRITE_BLOCKID 200
#define SDCARD_TEST_READ_BLOCKID 1000
#define SDCARD_TEST_NUM 10000
#define HASH_BUCKET_SIZE 50
#define CACHE_SIZE 500
#define SEGMENT_TYPE_SMALL_BITMAP_SIZE 32768
#define SEGMENT_TYPE_MEDIUM_BITMAP_SIZE 4096
#ifndef YCSB_TEST
#define SEGMENT_TYPE_BIG_BITMAP_SIZE 512
#else
#define SEGMENT_TYPE_BIG_BITMAP_SIZE 512
#endif
#define SEGMENT_TYPE_SMALL_START 64
#define SEGMENT_TYPE_MEDIUM_START 2097216
#define SEGMENT_TYPE_BIG_START 6291528
#define SEGMENT_TYPE_SMALL_CELL_SIZE 4096
#define SEGMENT_TYPE_MEDIUM_CELL_SIZE 65536
#ifndef YCSB_TEST
#define SEGMENT_TYPE_BIG_CELL_SIZE 2097152
#else
#define SEGMENT_TYPE_BIG_CELL_SIZE 65536000
#endif
#define PREFETCH_BLOCK_SERIALIZED_LENGTH 33
#define MAX_BRANCH_COUNT 4
#define PRELOAD_BLOCK_COUNT 10
#define PRELOAD_CHECK_INTERVAL_US 50
#define BLOCK_SIZE 512
#ifndef YCSB_TEST
#define BUFFER_SIZE 200
#else
#define BUFFER_SIZE 300000
#endif
#define CACHED_PAGE_COUNT 100
#define ROOTTABLE_TUPLE_SIZE sizeof(RootTable::TableTuple)
#define TABLE_ID_SIZE sizeof(uint64_t)
#define SYSTEM_TABLE_COUNT 1
#define ROOTTABLE_TABLE_ID 1
#define ROOTTABLE_SEGMENT_ID 1
#define ROOTTABLE_FIRST_BLOCK_ID 14680164
#define TABLE1_META_BLOCKID 1000
#define TABLE1_META_LOG_BLOCKID 10000
#define TABLE1_INDEX_BLOCKID 2000
#define TABLE1_INDEX_LOG_BLOCKID 20000
#define TABLE1_DATA_LOG_BLOCKID 30000
#define META_ENTRY_NUM 8
#define META_ENTRY_SIZE 64
#define BUFFER_MAX 4096
#define COMMON_TABLE_START_ID 101
#define YCSB_TABLE_ID 100
#define TABLE_NAME_LENGTH 32
#define PRIMARY_KEY_LENGTH 128
#define FILE_PATH_LENGTH 1024
#define FILE_NAME_LENGTH 128
#define USER_NAME_LENGTH 32
#define COLUMN_NAME_LENGTH 128
#define COLUMN_TYPE_NAME_LENGTH 32
#define PREFETCH_BLOCK_SERIALIZED_LENGTH 33
#define ROWMAP_SERIALIZED_LENGTH 42
#define SCHEMA_ENTRY_SERIALIZED_LENGTH 181
#define MAX_META_LENGTH 2048
#define MAX_PATH_LENGTH 128
const char METAPATH[MAX_PATH_LENGTH] = "../../data";
struct timespec diff(struct timespec start, struct timespec end);
class Queue {
public:
#ifdef IO_PROFILING
static struct timespec accumulate_io_times();
static struct timespec accumulate_io_submit_times();
#endif
#ifdef IO_PROFILING
static std::queue<struct timespec> io_times;
static std::queue<struct timespec> io_submit_times;
#endif
};
// A macro to disallow the copy constructor and operator= functions
#ifndef DISALLOW_COPY_AND_ASSIGN
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName &) = delete; \
TypeName &operator=(const TypeName &) = delete;
#endif
} // namespace LightTable
#endif // COMMON_H

View File

@ -0,0 +1,35 @@
/**
* @file CommonTable.h
* @brief CommonTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef COMMONTABLE_H
#define COMMONTABLE_H
#include "Table.h"
namespace LightTable {
class CommonTable : public Table {
public:
CommonTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex = false);
CommonTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
bool usePrimaryKeyIndex = false);
friend class MetaHandle;
protected:
DISALLOW_COPY_AND_ASSIGN(CommonTable);
};
} // namespace LightTable
#endif // COMMONTABLE_H

View File

@ -0,0 +1,61 @@
/**
* @file Driver.h
* @brief driver
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef DRIVER_H
#define DRIVER_H
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <atomic>
#include <iostream>
#include <thread>
#include "Common.h"
#include "Error.h"
// #include <mutex>
// #include <pthread.h>
// #include <condition_variable>
#ifdef __cplusplus
extern "C" {
#endif
uint32_t MakeTableStorage(const char *device_name);
#ifdef __cplusplus
}
#endif
namespace LightTable {
class Driver {
public:
static uint32_t driver_init(const char *device_name);
static void driver_cleanup(const char *device_name);
static uint32_t driver_read(uint8_t *buf, uint64_t blockID);
static uint32_t driver_write(uint8_t *buf, uint64_t blockID);
static uint32_t read(uint8_t *buf, uint64_t blockID);
static uint32_t write(uint8_t *buf, uint64_t blockID);
Driver(){};
#ifdef TABLE_STORAGE_CACHE
static BlockCache *blkCache;
#endif
};
} // namespace LightTable
#endif // DRIVER_H

View File

@ -0,0 +1,62 @@
/**
* @file Error.h
* @brief error number
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef ERROR_H
#define ERROR_H
namespace LightTable {
enum ErrorCode {
SUCCESS = 0,
ERROR_TABLE = 1,
SEGMENT_TYPE_ERROR = 2,
SEGMENT_STATUS_ERROR = 3,
SEGMENT_IS_EXHAUSTED = 4,
BUCKET_ITEM_NOT_FOUND = 5,
ADD_BUCKET_ITEM_ERROR = 6,
PREFETCH_BLOCK_NOT_FOUND = 7,
GET_AVAILABLE_BLOCK_ERROR = 8,
BLOCK_OCCUPIED = 9,
ADD_ROW_MAP_ENTRY_ERROR = 10,
ROW_MAP_ENTRY_NOT_FOUND = 11,
ADD_SCHEMA_ENTRY_ERROR = 12,
SCHEMA_ENTRY_NOT_FOUND = 13,
COLUMN_ITEM_NOT_FOUND = 14,
COLUMN_ID_NOT_CONTINUOUS = 15,
DELETE_PARIMARY_KEY_COLUMN = 16,
TABLE_NOT_FOUND = 17,
TABLE_TUPLE_NOT_FOUND = 18,
ADD_TABLE_TUPLE_ERROR = 19,
BUFFER_EMPTY = 20,
TYPE_INVALID = 21,
ROWLOCATION_INVALID = 22,
CREATE_SOCKET_ERROR = 23,
INET_PTON_ERROR = 24,
BIND_SOCKET_ERROR = 25,
LISTEN_SOCKET_ERROR = 26,
ACCEPT_SOCKET_ERROR = 27,
CONNECT_SOCKET_ERROR = 28,
SEND_MESSAGE_ERROR = 29,
EVENT_TYPE_NOT_DEFINED = 30,
SCHEMA_NOT_EXIST = 31,
OPEN_META_FILE_ERROR = 32,
INVALID_META_FILE = 33,
INVALID_MEMORY_BLOCK_ID = 34,
UPDATE_TUPLE_ERROR = 35,
INVALID_PAGE_ID = 36,
ADD_QUERY_CACHE_ERROR = 37,
TRIE_LEAF_EXIST = 38,
TRIE_LEAF_NOT_EXIST = 39,
TRIE_LEAF_NOT_FOUND = 40,
INVALID_SCAN_RANGE = 41,
INVALID_PRELOAD_TASK = 42,
};
} // namespace LightTable
#endif // ERROR_H

View File

@ -0,0 +1,59 @@
/**
* @file HashBucket.h
* @brief HashBucket
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef HASHBUCKET_H
#define HASHBUCKET_H
#include <stdint.h>
// #include <pthread.h>
#include <algorithm>
#include <map>
#include <iostream>
#include <utility>
// #include <mutex>
#include "Common.h"
#include "Error.h"
namespace LightTable {
typedef struct rowLocation {
uint64_t blockID;
uint64_t rowOffset;
} RowLocation;
class HashBucket {
public:
HashBucket();
uint32_t addBucketItem(uint64_t key, RowLocation rowLocation);
uint32_t deleteBucketItem(uint64_t key);
uint32_t updateRowLocation(uint64_t key, RowLocation rowLocation);
uint32_t getRowLocation(uint64_t key, RowLocation &rowLocation);
uint64_t getBucketSize();
uint32_t setBucketSize(uint64_t bucketSize);
uint32_t getAllBucketItems(std::map<uint64_t, RowLocation> &bucketItems);
private:
std::map<uint64_t, RowLocation> bucketItems;
uint64_t bucketSize;
// pthread_mutex_t mtx;
DISALLOW_COPY_AND_ASSIGN(HashBucket);
};
} // namespace LightTable
#endif // HASHBUCKET_H

View File

@ -0,0 +1,63 @@
/**
* @file HashTable.h
* @brief HashTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef HASHTABLE_H
#define HASHTABLE_H
#include <stdint.h>
#include <algorithm>
#include <vector>
#include <iostream>
#include <string>
#include "Common.h"
#include "HashBucket.h"
namespace LightTable {
class HashTable {
public:
HashTable();
~HashTable();
uint64_t getMapKey(const char *primaryKey);
uint32_t hashToBucket(uint64_t key);
uint32_t getRowLocation(uint32_t bucketID, uint64_t key,
RowLocation &rowLocation);
uint32_t addItem(uint32_t bucketID, uint64_t key, RowLocation rowLocation);
uint32_t deleteItem(uint32_t bucketID, uint64_t key);
uint32_t updateItem(uint32_t bucketID, uint64_t key, RowLocation rowLocation);
uint64_t getItemCount();
uint32_t getAllBuckets(std::vector<HashBucket *> &buckets);
uint32_t refreshRowLocations(uint64_t blockID, uint64_t deleteOffset,
uint64_t rowSize,
std::map<uint64_t, RowLocation> &locationItems);
uint32_t clear();
private:
std::vector<HashBucket *> buckets;
DISALLOW_COPY_AND_ASSIGN(HashTable);
public:
friend class MetaHandle;
};
} // namespace LightTable
#endif // HASHTABLE_H

View File

@ -0,0 +1,101 @@
/**
* @file PrefetchBlockManager.h
* @brief PrefetchBlockManager
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef PREFETCHBLOCKMANAGER_H
#define PREFETCHBLOCKMANAGER_H
#include <stdint.h>
#include <time.h>
#include <stdlib.h>
// #include <pthread.h>
#include <map>
#include <iostream>
#include <algorithm>
#include <chrono>
#include <utility>
// #include <mutex>
#include "Error.h"
#include "Common.h"
#include "SegmentManager.h"
namespace LightTable
{
enum PrefetchBlockStatus
{
PREFETCH_BLOCK_STATUS_IDLE = 0,
PREFETCH_BLOCK_STATUS_BUSY,
};
class PrefetchBlockManager
{
public:
typedef struct prefetchBlockManagerEntry
{
uint64_t blockID;
uint64_t currentOffset;
bool isOccupied;
bool isCached;
uint64_t pageID;
}PrefetchBlockManagerEntry;
PrefetchBlockManager(uint64_t firstBlockID, uint64_t segmentID,
SegmentType segmentType);
~PrefetchBlockManager();
uint64_t blockAllocate(uint64_t rowSize);
uint64_t getNextBlock();
uint32_t setBlockState(uint64_t blockID, bool isOccupied);
uint32_t setOffset(uint64_t blockID, uint64_t currentOffset);
uint32_t getOffset(uint64_t blockID, uint64_t &currentOffset);
uint32_t advanceOffset(uint64_t blockID, uint64_t rowSize);
uint32_t isOccupied(uint64_t blockID, bool &status);
uint32_t getPrefetchBlockEntry(uint64_t blockID,
PrefetchBlockManagerEntry &entry);
uint32_t getAllPrefetchBlockEntrys(std::map<uint64_t,
PrefetchBlockManagerEntry> &entrys);
uint32_t lockBlock(uint64_t blockID);
uint32_t unlockBlock(uint64_t blockID);
uint32_t setIsCached(uint64_t blockID, bool isCached, uint64_t pageID);
bool isCached(uint64_t blockID, uint64_t &pageID);
private:
std::map<uint64_t, PrefetchBlockManagerEntry> prefetchBlocks;
const uint64_t segmentID;
const SegmentType segmentType;
const uint64_t firstBlockID;
uint64_t blockCount;
uint64_t currentID;
DISALLOW_COPY_AND_ASSIGN(PrefetchBlockManager);
public:
friend class QueryCache;
friend class MetaHandle;
};
} // namespace LightTable
#endif // PREFETCHBLOCKMANAGER_H

View File

@ -0,0 +1,57 @@
/**
* @file RWLock.h
* @brief RWLock
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef RWLOCK_H
#define RWLOCK_H
#include <stdint.h>
#include <stdio.h>
#include <atomic>
namespace LightTable {
using version_lock_t = uint8_t;
#define LOAD_ORDER std::memory_order_acquire
#define STORE_ORDER std::memory_order_release
static_assert(sizeof(version_lock_t) == 1, "Lock must be 1 byte.");
static constexpr version_lock_t CLIENT_BIT = 0b10000000;
static constexpr version_lock_t NO_CLIENT_BIT = 0b01111111;
static constexpr version_lock_t USED_BIT = 0b01000000;
static constexpr version_lock_t UNLOCKED_BIT = 0b11111110;
#define IS_LOCKED(lock) ((lock)&1)
class RWLock {
std::atomic<version_lock_t> version_lock;
public:
bool lock(const bool blocking = true) {
version_lock_t lock_value = version_lock.load(LOAD_ORDER);
// Compare and swap until we are the thread to set the lock bit
lock_value &= UNLOCKED_BIT;
while (!version_lock.compare_exchange_weak(lock_value, lock_value + 1)) {
lock_value &= UNLOCKED_BIT;
if (!blocking) {
return false;
}
}
return true;
}
void unlock() {
const version_lock_t current_version = version_lock.load(LOAD_ORDER);
version_lock_t new_version = (current_version + 1) % USED_BIT;
new_version |= USED_BIT;
new_version |= (current_version & CLIENT_BIT);
version_lock.store(new_version, STORE_ORDER);
}
};
} // namespace LightTable
#endif // HASHTABLE_H

View File

@ -0,0 +1,75 @@
/**
* @file RootTable.h
* @brief RootTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef ROOTTABLE_H
#define ROOTTABLE_H
#include <stddef.h>
#include <time.h>
#include <algorithm>
#include <iostream>
#include <map>
#include "Common.h"
#include "SystemTable.h"
namespace LightTable {
class Buffer;
class RootTable : public SystemTable {
public:
typedef struct tableTuple {
uint64_t tableID;
char tableName[TABLE_NAME_LENGTH];
uint64_t firstBlockID;
uint64_t segmentID;
SegmentType segmentType;
char user[USER_NAME_LENGTH];
struct tm time;
} TableTuple;
RootTable(std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex)
: SystemTable(ROOTTABLE_TABLE_ID, "RootTable", ROOTTABLE_FIRST_BLOCK_ID,
ROOTTABLE_SEGMENT_ID, SEGMENT_TYPE_BIG, schemaEntrys,
usePrimaryKeyIndex) {}
RootTable(bool usePrimaryKeyIndex = false);
uint32_t tableNameToTableID(const char *tableName, uint64_t &tableID);
uint32_t deleteTableInfo(uint64_t tableID);
uint32_t appendTableInfo(TableTuple tableTuple);
uint32_t getTableInfo(uint64_t tableID, TableTuple &tableTupleResult);
uint32_t modifyTableInfo(uint64_t tableID, TableTuple tableTuple);
uint32_t getTableTuple(uint64_t tableID, TableTuple &tableTuple);
uint32_t serializeRow(const TableTuple tableTuple, uint8_t *buf);
uint32_t deserializeRow(const uint8_t *buf, TableTuple *tableTuple);
uint64_t getTableCount();
private:
std::map<uint64_t, TableTuple> tableTupleMap;
DISALLOW_COPY_AND_ASSIGN(RootTable);
public:
friend class MetaHandle;
};
} // namespace LightTable
#endif // ROOTTABLE_H

View File

@ -0,0 +1,57 @@
/**
* @file RowMap.h
* @brief
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef ROWMAP_H
#define ROWMAP_H
#include <stdint.h>
#include <algorithm>
#include <iostream>
#include <map>
#include <string>
#include "Common.h"
#include "Error.h"
namespace LightTable {
class RowMap {
public:
typedef struct rowMapEntry {
uint64_t key;
uint64_t blockID;
uint64_t rowOffset;
} RowMapEntry;
RowMap();
uint32_t appendEntry(uint64_t key, uint64_t blockID, uint64_t rowOffset);
uint32_t deleteEntry(uint64_t key);
uint32_t alterEntry(uint64_t key, uint64_t blockID, uint64_t rowOffset);
uint32_t queryEntry(uint64_t key, RowMapEntry &rowMapEntry);
uint32_t queryAllEntry(std::map<uint64_t, RowMapEntry> &allRowMapEntrys);
uint64_t getRowCount();
private:
std::map<uint64_t, RowMapEntry> rowMapEntrys;
DISALLOW_COPY_AND_ASSIGN(RowMap);
public:
friend class MetaHandle;
};
} // namespace LightTable
#endif // ROWMAP_H

View File

@ -0,0 +1,92 @@
/**
* @file Schema.h
* @brief table schema
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef SCHEMA_H
#define SCHEMA_H
#include <stdint.h>
#include <string.h>
#include <algorithm>
#include <iostream>
#include <map>
#include <string>
#include "Common.h"
#include "Error.h"
namespace LightTable {
class Schema {
public:
typedef struct schemaEntry {
uint64_t columnID;
char columnName[COLUMN_NAME_LENGTH];
char type[COLUMN_TYPE_NAME_LENGTH];
uint32_t length;
bool isPrimaryKey;
bool isEmpty;
schemaEntry &operator=(schemaEntry &another) {
this->columnID = another.columnID;
memcpy(this->columnName, another.columnName, COLUMN_NAME_LENGTH);
memcpy(this->type, another.type, COLUMN_TYPE_NAME_LENGTH);
this->length = another.length;
this->isPrimaryKey = another.isPrimaryKey;
this->isEmpty = another.isEmpty;
return *this;
}
} SchemaEntry;
Schema();
Schema(std::vector<SchemaEntry> schemaEntrys);
uint32_t appendEntry(SchemaEntry &schemaEntry);
uint32_t deleteEntry(uint64_t columnID);
uint32_t alterEntry(uint64_t columnID, SchemaEntry schemaEntry);
uint32_t getColumnID(const char *columnName, uint64_t &columnID);
uint32_t queryEntry(uint64_t columnID, Schema::SchemaEntry &schemaEntry);
uint32_t queryAllEntry(std::map<uint64_t, SchemaEntry> &allSchemaEntrys);
uint64_t getEntrySize();
void setEntrySize(uint64_t entrySize);
uint32_t getPrimaryKeyLength();
uint64_t getColumnCount();
void setColumnCount(uint64_t columnCount);
Schema::SchemaEntry &getPrimaryKeySchema();
uint32_t drop();
private:
std::map<uint64_t, SchemaEntry> schemaEntrys;
SchemaEntry primaryKeySchema;
uint64_t columnCount;
uint64_t entrySize;
DISALLOW_COPY_AND_ASSIGN(Schema);
public:
friend class MetaHandle;
};
} // namespace LightTable
#endif // SCHEMA_H

View File

@ -0,0 +1,59 @@
/**
* @file SegmentManager.h
* @brief SegmentManager
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef SEGMENTMANAGER_H
#define SEGMENTMANAGER_H
#include <stdint.h>
#include "Common.h"
#include "Error.h"
#include "Driver.h"
#include "Common.h"
namespace LightTable {
enum SegmentType {
SEGMENT_TYPE_SMALL = 101,
SEGMENT_TYPE_MEDIUM,
SEGMENT_TYPE_BIG,
};
enum SegmentStatus {
SEGMENT_STATUS_IDLE = 0,
SEGMENT_STATUS_BUSY,
};
class SegmentManager {
public:
SegmentManager();
~SegmentManager();
uint32_t setBitmap(SegmentType segmentType, uint64_t segmentID,
SegmentStatus segmentStatus);
uint32_t getIdleSegment(SegmentType segmentType, uint64_t &segmentID,
uint64_t &firstBlockID);
private:
uint32_t setBit(uint64_t segmentID, uint8_t *buf, uint64_t segmentStart,
SegmentStatus segmentStatus);
uint32_t findFirstIdle(uint8_t *buf, uint64_t segmentNum,
uint64_t &segmentID);
uint8_t *bigBitmap;
uint8_t *mediumBitmap;
uint8_t *smallBitmap;
DISALLOW_COPY_AND_ASSIGN(SegmentManager);
};
} // namespace LightTable
#endif // SEGMENTMANAGER_H

View File

@ -0,0 +1,33 @@
/**
* @file SystemTable.h
* @brief SystemTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef SYSTEMTABLE_H
#define SYSTEMTABLE_H
#include "Table.h"
namespace LightTable {
class SystemTable : public Table {
public:
SystemTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex);
SystemTable(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
bool usePrimaryKeyIndex);
protected:
DISALLOW_COPY_AND_ASSIGN(SystemTable);
};
} // namespace LightTable
#endif // SYSTEMTABLE_H

View File

@ -0,0 +1,147 @@
/**
* @file Table.h
* @brief Table
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef TABLE_H
#define TABLE_H
#include <stdint.h>
#include <string.h>
#include <stddef.h>
#include <algorithm>
#include <ctime>
#include <iostream>
#include <queue>
#include <string>
#include <vector>
#include "Common.h"
#include "HashTable.h"
#include "RowMap.h"
#include "Schema.h"
#include "Driver.h"
#include "UndoLogEntry.h"
#include "SegmentManager.h"
#include "PrefetchBlockManager.h"
namespace LightTable {
#ifdef TABLE_STORAGE_SECOND_INDEX
// TODO: second index
#endif
class Buffer;
class Table {
typedef struct currentLocation {
uint64_t blockID;
int32_t emptySlotNum;
} currentLocation;
typedef struct metaLogEntry {
uint64_t tableID;
uint64_t tupleNum;
std::time_t t;
} metaLogEntry;
public:
Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex = false);
Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
bool usePrimaryKeyIndex = false);
uint32_t drop();
uint32_t truncate();
uint32_t insertRow(const uint8_t *rowData);
uint32_t insertRow_logging(const uint8_t *rowData);
uint32_t deleteRow(const char *primaryKey);
uint32_t updateRow(const char *primaryKey, const uint8_t *rowData);
uint32_t updateColumnItem(const char *primaryKey, const char *columnName,
const uint8_t *columnData);
uint32_t updateColumnItem_logging(const char *primaryKey,
const char *columnName,
const uint8_t *columnData);
uint32_t selectRow(const char *primaryKey, uint8_t *rowData);
uint32_t selectColumnItem(const char *primaryKey, const char *columnName,
uint8_t *columnData);
uint64_t getRowCount();
uint64_t getTableID();
char *getTableName();
uint32_t setTableID(uint64_t tableID);
uint32_t setTableName(const char *tableName);
Schema &getSchema();
HashTable &getHashTable();
protected:
uint32_t getColumnID(const char *columnName, uint64_t &columnID);
uint32_t parsePrimaryKey(const uint8_t *rowData, char *primaryKeyData);
uint32_t locateRow(const char *primaryKey, RowLocation &rowLocation);
uint32_t getRowFromBlock(const char *primaryKey, uint8_t *rowData,
const uint8_t *blockData,
const RowLocation rowLocation);
uint32_t selectRow(const char *primaryKey, uint8_t *rowData,
RowLocation &rowLocation);
uint32_t deleteRowData(uint64_t key, RowLocation rowLocation);
uint32_t getColumnOffset(const char *columnName, uint64_t &columnID,
uint32_t &columnLength, uint64_t &offset);
uint64_t tableID;
char tableName[TABLE_NAME_LENGTH];
Schema schema;
RowMap rowMap;
HashTable hashTable;
bool usePrimaryKeyIndex;
uint64_t tupleNum;
std::time_t t;
currentLocation indexLocation;
currentLocation metaLogLocation;
currentLocation indexLogLocation;
currentLocation dataLogLocation;
uint8_t indexData[BLOCK_SIZE];
uint8_t metaData[BLOCK_SIZE];
uint8_t metaLog[BLOCK_SIZE];
uint8_t indexLog[BLOCK_SIZE];
uint8_t dataLog[BLOCK_SIZE];
PrefetchBlockManager prefetchBlockManager;
DISALLOW_COPY_AND_ASSIGN(Table);
public:
friend class DaemonProcess;
friend class MetaHandle;
friend class QueryCache;
};
} // namespace LightTable
#endif // TABLE_H

View File

@ -0,0 +1,145 @@
/**
* @file TableStorage.h
* @brief TableStorage
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef DAEMONPROCESS_H
#define DAEMONPROCESS_H
#include <stdint.h>
#include <stdio.h>
#include <time.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <algorithm>
#include <map>
#include <iomanip>
#include <iostream>
#include <thread>
// #include <sys/types.h>
// #include <sys/socket.h>
// #include <arpa/inet.h>
// #include <netinet/in.h>
// #include <netinet/tcp.h>
#include "RootTable.h"
#include "SegmentManager.h"
#include "Common.h"
#include "CommonTable.h"
#include "Error.h"
#include "Driver.h"
#include "Table.h"
#define TABLE_DEBUG_ENABLE
#define PROJECT_NAME "TanleStorage"
#ifndef TABLE_PRINT
#define TABLE_PRINT(...) printf(__VA_ARGS__)
#endif
#define TABLE_LOG_PREFIX1() \
TABLE_PRINT("[\e[1;34m%s\033[0m] \e[1;31m%s\033[0m", PROJECT_NAME, \
__FUNCTION__)
#define TABLE_LOG_PREFIX2() TABLE_PRINT(" ")
#define TABLE_LOG_PREFIX() \
TABLE_LOG_PREFIX1(); \
TABLE_LOG_PREFIX2()
#ifdef TABLE_DEBUG_ENABLE
#define TABLE_DEBUG(...) \
TABLE_LOG_PREFIX(); \
TABLE_PRINT("(\e[1;32m%s:%d\033[0m) ", __FILE__, __LINE__); \
TABLE_PRINT(__VA_ARGS__)
#else
#define TABLE_DEBUG(...)
#endif
namespace LightTable {
enum EventType {
CREATE_TABLE_EVENT = 10001,
INSERT_ROW_EVENT,
DELETE_ROW_EVENT,
SELECT_ROW_EVENT,
#ifdef YCSB_TEST
SCAN_EVENT,
#endif
UPDATE_ROW_EVENT,
#ifdef YCSB_TEST
UPDATE_COLUMNS_EVENT,
#endif
DROP_TABLE_EVENT,
RESPOND_CLIENT_REQUEST,
STOP_SERVICE_EVENT,
#ifdef YCSB_TEST
UNKNOWN_EVENT,
#endif
};
typedef struct message {
uint32_t size;
EventType eventType;
uint8_t data[NETWORK_MESSAGE_DATA_MAX_SIZE];
} Message;
class TableStorage {
public:
static TableStorage *getInstance(const char *ipAddress,
const uint32_t portNum,
const char *path = METAPATH) {
static TableStorage *daemonProcess;
// To determine whether the first call.
if (daemonProcess == NULL) {
daemonProcess = new TableStorage(ipAddress, portNum, path);
}
return daemonProcess;
}
uint32_t createTable(const char *tableName, SegmentType segmentType,
std::vector<Schema::SchemaEntry> &schema);
uint32_t dropTable(const char *tableName);
uint32_t insertTuple(const char *tableName, const uint8_t *row);
uint32_t deleteTuple(const char *tableName, const char *primaryKey);
uint32_t updateTuple(const char *tableName, const char *primaryKey,
const char *columnName, const uint8_t *value);
uint32_t selectTuple(const char *tableName, const char *primaryKey,
uint8_t *row);
uint32_t initTableStorage();
uint32_t closeTableStorage();
private:
TableStorage(const char *ipAddress, const uint32_t portNum, const char *path);
~TableStorage();
RootTable *rootTable;
std::map<uint64_t, Table *> tableMap;
SegmentManager *segmentManager;
const char *LightTableIPAddress;
const uint32_t LightTablePortNum;
// int32_t socketfd;
// MetaHandle metaHandle;
// Buffer *buf;
// QueryCache *queryCache;
// BlockPreload *blockPreload;
friend class Benchmark;
};
} // namespace LightTable
#endif // DAEMONPROCESS_H

View File

@ -0,0 +1,81 @@
/**
* @file ThreadSafeQueue.h
* @brief ThreadSafeQueue
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include <pthread.h>
#include <iostream>
#include "Common.h"
using namespace std;
namespace LightTable {
template <class object>
class ThreadSafeQueue {
private:
pthread_mutex_t m_lock;
int m_front;
int m_rear;
object m_data[BUFFER_SIZE];
public:
ThreadSafeQueue() : m_front(0), m_rear(0) {
pthread_mutex_init(&m_lock, NULL);
}
bool EnQueue(object data) {
pthread_mutex_lock(&m_lock);
if (isFull()) {
// cout<<"The queue is full!"<<endl;
pthread_mutex_unlock(&m_lock);
return false;
}
m_data[m_rear] = data;
m_rear = (m_rear + 1) % BUFFER_SIZE;
pthread_mutex_unlock(&m_lock);
return true;
}
bool DeQueue(object& data) {
pthread_mutex_lock(&m_lock);
if (isEmpty()) {
pthread_mutex_unlock(&m_lock);
return false;
}
data = m_data[m_front];
m_front = (m_front + 1) % BUFFER_SIZE;
pthread_mutex_unlock(&m_lock);
return true;
}
bool top(object& data) {
pthread_mutex_lock(&m_lock);
if (isEmpty()) {
pthread_mutex_unlock(&m_lock);
return false;
}
data = m_data[m_front];
pthread_mutex_unlock(&m_lock);
return true;
}
bool isFull() {
if ((m_rear + 1) % BUFFER_SIZE == m_front) return true;
return false;
}
bool isEmpty() {
if (m_rear == m_front) return true;
return false;
}
~ThreadSafeQueue() { pthread_mutex_destroy(&m_lock); }
};
} // namespace LightTable

View File

@ -0,0 +1,49 @@
/**
* @file UndoLogEntry.h
* @brief UndoLogEntry
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef UNDOLOGENTRY_H
#define UNDOLOGENTRY_H
#include <stdint.h>
#include "BufferItem.h"
#include "Common.h"
#include "Error.h"
namespace LightTable {
class UndoLogEntry : public BufferItem {
public:
UndoLogEntry(uint64_t tableID, const char *primaryKey, uint64_t keySize,
char *tuple, uint64_t tupleSize, uint64_t blockID,
uint64_t offset);
static bool getIsExpired(UndoLogEntry *logEntry);
void expireUndoLog();
uint32_t getTuple(uint64_t tableID, const char *primaryKey, char *tuple,
uint64_t &blockID, uint64_t &offset);
char *getPrimaryKey();
~UndoLogEntry();
private:
char *tuple;
char *primaryKey;
uint64_t blockID;
uint64_t offset;
uint64_t tupleSize;
uint64_t keySize;
bool isExpired;
DISALLOW_COPY_AND_ASSIGN(UndoLogEntry);
};
} // namespace LightTable
#endif // UNDOLOGENTRY_H

View File

@ -0,0 +1,41 @@
/**
* @file integer.h
* @brief integer
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#ifndef _FF_INTEGER
#define _FF_INTEGER
#ifdef _WIN32 /* FatFs development platform */
#include <tchar.h>
#include <windows.h>
typedef unsigned __int64 QWORD;
#else /* Embedded platform */
/* These types MUST be 16-bit or 32-bit */
typedef int INT;
typedef unsigned int UINT;
/* This type MUST be 8-bit */
typedef unsigned char BYTE;
/* These types MUST be 16-bit */
typedef short SHORT;
typedef unsigned short WORD;
typedef unsigned short WCHAR;
/* These types MUST be 32-bit */
typedef long LONG;
typedef unsigned long DWORD;
/* This type MUST be 64-bit (Remove this for C89 compatibility) */
typedef unsigned long long QWORD;
#endif
#endif

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 132 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 92 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 149 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 177 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 227 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 144 KiB

View File

@ -0,0 +1,80 @@
/**
* @file Common.cc
* @brief Common
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "Common.h"
namespace LightTable {
int cacheReadNum = 0;
int cacheWriteNum = 0;
std::chrono::steady_clock::time_point start_time =
std::chrono::steady_clock::now();
std::chrono::steady_clock::time_point end_time =
std::chrono::steady_clock::now();
double diff_sdcard_read = 0.0;
double diff_sdcard_write = 0.0;
struct timespec diff(struct timespec start, struct timespec end) {
struct timespec temp;
if ((end.tv_nsec - start.tv_nsec) < 0) {
temp.tv_sec = end.tv_sec - start.tv_sec - 1;
temp.tv_nsec = 1000000000 + end.tv_nsec - start.tv_nsec;
} else {
temp.tv_sec = end.tv_sec - start.tv_sec;
temp.tv_nsec = end.tv_nsec - start.tv_nsec;
}
return temp;
}
#ifdef IO_PROFILING
std::queue<struct timespec> Queue::io_times;
std::queue<struct timespec> Queue::io_submit_times;
#endif
#ifdef IO_PROFILING
struct timespec Queue::accumulate_io_times() {
struct timespec sum;
sum.tv_sec = 0;
sum.tv_nsec = 0;
while (!io_times.empty()) {
sum.tv_sec += io_times.front().tv_sec;
if (1000000000 <= sum.tv_nsec + io_times.front().tv_nsec) {
sum.tv_sec++;
sum.tv_nsec = sum.tv_nsec + io_times.front().tv_nsec - 1000000000;
} else {
sum.tv_nsec += io_times.front().tv_nsec;
}
io_times.pop();
}
return sum;
}
struct timespec Queue::accumulate_io_submit_times() {
struct timespec sum;
sum.tv_sec = 0;
sum.tv_nsec = 0;
while (!io_submit_times.empty()) {
sum.tv_sec += io_submit_times.front().tv_sec;
if (1000000000 <= sum.tv_nsec + io_submit_times.front().tv_nsec) {
sum.tv_sec++;
sum.tv_nsec = sum.tv_nsec + io_submit_times.front().tv_nsec - 1000000000;
} else {
sum.tv_nsec += io_submit_times.front().tv_nsec;
}
io_submit_times.pop();
}
return sum;
}
#endif
} // namespace LightTable

View File

@ -0,0 +1,92 @@
/**
* @file Driver.cc
* @brief Driver
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "Driver.h"
extern "C" {
#include "ff.h"
#include "ffconf.h"
#include <diskio.h>
#include <rtdef.h>
}
/* Definitions of physical drive number for each drive */
#define DEV_RAM 0 /* Example: Map Ramdisk to physical drive 0 */
#define DEV_MMC 1 /* Example: Map MMC/SD card to physical drive 1 */
#define DEV_USB 2 /* Example: Map USB MSD to physical drive 2 */
static rt_device_t disk[FF_VOLUMES] = {0};
namespace LightTable {
uint32_t Driver::write(uint8_t *buf, uint64_t blockID) {
rt_size_t result;
rt_device_t device = disk[0];
#ifdef TABLE_STORAGE_CACHE
// TODO: cache
#endif
if (rt_device_write(device, blockID, buf, 1) == 1)
return RES_OK;
else
return RES_ERROR;
}
uint32_t Driver::read(uint8_t *buf, uint64_t blockID) {
rt_size_t result;
rt_device_t device = disk[0];
#ifdef TABLE_STORAGE_CACHE
// TODO: cache
#endif
if (rt_device_read(device, blockID, buf, 1) == 1) {
return RES_OK;
}
return RES_OK;
}
void Driver::driver_cleanup(const char *device_name) {
/* clean device */
return;
}
uint32_t Driver::driver_init(const char *device_name) {
/* init device */
/* note: initialization of the device is done at the MakeTableStorage*/
return 0;
}
} // namespace LightTable
uint32_t MakeTableStorage(const char *device_name) {
rt_device_t dev_id;
/* open specific device */
if (device_name == NULL) {
/* which is a non-device filesystem mount */
dev_id = NULL;
} else if ((dev_id = rt_device_find(device_name)) == NULL) {
/* no this device */
rt_set_errno(-ENODEV);
return -1;
}
/* open device, but do not check the status of device */
if (dev_id != NULL) {
if (rt_device_open(dev_id, RT_DEVICE_OFLAG_RDWR) != RT_EOK) {
return -1;
}
}
disk[0] = dev_id;
return 0;
}

View File

@ -0,0 +1,22 @@
ifndef LIGHTTABLE_DIR
LIGHTTABLE_DIR = ../..
endif
SRC_FILES := $(wildcard *.cc)
INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I. -I$(LIGHTTABLE_DIR)/spdk/include
SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1)))
CXXFLAGS += -Werror -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter \
-Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \
$(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \
-fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \
-D_FORTIFY_SOURCE=2 -std=c++11
CXXFLAGS += -DYCSB_TEST
CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \
-Wl,-z,noexecstack \
-Wl,--whole-archive -Wl,--no-whole-archive
include $(KERNEL_ROOT)/compiler.mk

View File

@ -0,0 +1,17 @@
/**
* @file BufferItem.cc
* @brief BufferItem
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "BufferItem.h"
namespace LightTable {
BufferItem::BufferItem(uint64_t tableID) : tableID(tableID) {}
BufferItem::~BufferItem() {}
} // namespace LightTable

View File

@ -0,0 +1,23 @@
ifndef LIGHTTABLE_DIR
LIGHTTABLE_DIR = ../..
endif
SRC_FILES := $(wildcard *.cc)
INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I.
SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1)))
CXXFLAGS += -Werror -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter \
-Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \
$(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \
-fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \
-D_FORTIFY_SOURCE=2 -std=c++11
CXXFLAGS += -DYCSB_TEST
CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \
-Wl,-z,noexecstack \
-Wl,--whole-archive -Wl,--no-whole-archive
include $(KERNEL_ROOT)/compiler.mk

View File

@ -0,0 +1,184 @@
/**
* @file TableStorage.cc
* @brief TableStorage
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "TableStorage.h"
// #include "ThreadPool.h"
// #include "Buffer.h"
// static struct timespec time1, time2, time3;
// static uint32_t coreNum = 0;
namespace LightTable {
TableStorage::TableStorage(const char *ipAddress, const uint32_t portNum,
const char *path)
: // metaHandle(path),
LightTableIPAddress(ipAddress),
LightTablePortNum(portNum) {
segmentManager = new SegmentManager();
rootTable = new RootTable();
std::cout << "INIT TableStorage SUCCESS" << "\n";
// threadPool = ThreadPool::getInstance(SYSTEM_MAX_CORES);
// buf = Buffer::getInstance();
// queryCache = QueryCache::getInstance(&tableMap);
// blockPreload = BlockPreload::getInstance();
// blockPreload->setParameters(queryCache);
// buf->setParameters(&metaHandle, rootTable);
}
uint32_t TableStorage::createTable(const char *tableName,
SegmentType segmentType,
std::vector<Schema::SchemaEntry> &schema) {
uint64_t segmentID = 0;
uint64_t firstBlockID = 0;
uint64_t tableID =
tableMap.size() + COMMON_TABLE_START_ID - SYSTEM_TABLE_COUNT;
uint32_t err =
segmentManager->getIdleSegment(segmentType, segmentID, firstBlockID);
if (err != SUCCESS) {
return err;
}
CommonTable *commonTable = new CommonTable(tableID, tableName, firstBlockID,
segmentID, segmentType, schema);
std::pair<std::map<uint64_t, Table *>::iterator, bool> ret;
ret = tableMap.insert(std::pair<uint64_t, Table *>(tableID, commonTable));
if (false == ret.second) {
return ERROR_TABLE;
}
time_t timeNow = time(NULL);
struct tm *now = localtime(&timeNow);
RootTable::TableTuple tuple;
tuple.tableID = tableID;
memcpy(tuple.tableName, tableName, TABLE_NAME_LENGTH);
tuple.firstBlockID = firstBlockID;
tuple.segmentType = segmentType;
tuple.segmentID = segmentID;
memcpy(tuple.user, "user", 5);
tuple.time = *now;
err = rootTable->appendTableInfo(tuple);
if (err == SUCCESS) std::cout << "create table: " << tableName << std::endl;
return err;
}
uint32_t TableStorage::dropTable(const char *tableName) {
uint64_t tableID = 0;
uint32_t result = rootTable->tableNameToTableID(tableName, tableID);
if (SUCCESS != result) return result;
std::map<uint64_t, Table *>::iterator iter = tableMap.find(tableID);
Table *commonTable = iter->second;
commonTable->drop();
tableMap.erase(tableID);
result = rootTable->deleteTableInfo(tableID);
if (result == SUCCESS) std::cout << "drop table: " << tableName << std::endl;
return result;
}
uint32_t TableStorage::insertTuple(const char *tableName, const uint8_t *row) {
uint64_t tableID = 0;
uint32_t result = 0;
#ifndef SDCARD_TEST
result = rootTable->tableNameToTableID(tableName, tableID);
if (SUCCESS != result) {
return result;
}
#else
tableID = YCSB_TABLE_ID;
#endif
std::map<uint64_t, Table *>::iterator iter = tableMap.find(tableID);
Table *commonTable = iter->second;
#ifndef TABLE_STORAGE_LOGGING
result = commonTable->insertRow(row);
#else
result = commonTable->insertRow_logging(row);
#endif
return result;
}
uint32_t TableStorage::deleteTuple(const char *tableName,
const char *primaryKey) {
uint64_t tableID = 0;
uint32_t result = rootTable->tableNameToTableID(tableName, tableID);
if (SUCCESS != result) return result;
std::map<uint64_t, Table *>::iterator iter = tableMap.find(tableID);
Table *commonTable = iter->second;
result = commonTable->deleteRow(primaryKey);
if (result == SUCCESS)
std::cout << "delete a tuple in " << tableName << " success!" << std::endl;
return result;
}
uint32_t TableStorage::updateTuple(const char *tableName,
const char *primaryKey,
const char *columnName,
const uint8_t *value) {
uint64_t tableID = 0;
uint32_t result = rootTable->tableNameToTableID(tableName, tableID);
if (SUCCESS != result) return result;
std::map<uint64_t, Table *>::iterator iter = tableMap.find(tableID);
Table *commonTable = iter->second;
#ifndef TABLE_STORAGE_LOGGING
result = commonTable->updateColumnItem(primaryKey, columnName, value);
#else
result = commonTable->updateColumnItem_logging(primaryKey, columnName, value);
#endif
return result;
}
uint32_t TableStorage::selectTuple(const char *tableName,
const char *primaryKey, uint8_t *row) {
uint64_t tableID = 0;
uint32_t result = 0;
#ifndef SDCARD_TEST
result = rootTable->tableNameToTableID(tableName, tableID);
if (SUCCESS != result) return result;
#else
tableID = YCSB_TABLE_ID;
#endif
std::map<uint64_t, Table *>::iterator iter = tableMap.find(tableID);
Table *commonTable = iter->second;
result = commonTable->selectRow(primaryKey, row);
return result;
}
uint32_t TableStorage::initTableStorage() {
return SUCCESS;
}
uint32_t TableStorage::closeTableStorage() {
return SUCCESS;
}
TableStorage::~TableStorage() {
for (auto ite = tableMap.begin(); ite != tableMap.end(); ++ite)
delete (ite->second);
tableMap.clear();
delete (rootTable);
delete (segmentManager);
std::cout << "close TableStorage success!\n";
}
} // namespace LightTable

View File

@ -0,0 +1,54 @@
/**
* @file UndoLogEntry.cc
* @brief UndoLogEntry
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "UndoLogEntry.h"
#include <string.h>
namespace LightTable {
UndoLogEntry::UndoLogEntry(uint64_t tableID, const char *primaryKey,
uint64_t keySize, char *tuple, uint64_t tupleSize,
uint64_t blockID, uint64_t offset)
: BufferItem(tableID),
blockID(blockID),
offset(offset),
tupleSize(tupleSize),
keySize(keySize),
isExpired(false) {
memcpy(this->tuple, tuple, tupleSize);
memcpy(this->primaryKey, primaryKey, keySize);
}
UndoLogEntry::~UndoLogEntry() {
delete[] primaryKey;
delete[] tuple;
}
uint32_t UndoLogEntry::getTuple(uint64_t tableID, const char *primaryKey,
char *tuple, uint64_t &blockID,
uint64_t &offset) {
if (this->tableID != tableID || strcmp(this->primaryKey, primaryKey) != 0) {
return ERROR_TABLE;
}
memcpy(tuple, this->tuple, tupleSize);
blockID = this->blockID;
offset = this->offset;
return SUCCESS;
}
void UndoLogEntry::expireUndoLog() { isExpired = true; }
char *UndoLogEntry::getPrimaryKey() { return this->primaryKey; }
bool UndoLogEntry::getIsExpired(UndoLogEntry *logEntry) {
return logEntry->isExpired;
}
} // namespace LightTable

View File

@ -0,0 +1,31 @@
/**
* @file CommonTable.cc
* @brief CommonTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "CommonTable.h"
namespace LightTable {
CommonTable::CommonTable(uint64_t tableID, const char *tableName,
uint64_t firstBlockID, uint64_t segmentID,
SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex)
: Table(tableID, tableName, firstBlockID, segmentID, segmentType,
schemaEntrys, usePrimaryKeyIndex)
{}
CommonTable::CommonTable(uint64_t tableID, const char *tableName,
uint64_t firstBlockID, uint64_t segmentID,
SegmentType segmentType, bool usePrimaryKeyIndex)
: Table(tableID, tableName, firstBlockID, segmentID, segmentType,
usePrimaryKeyIndex)
{}
} // namespace LightTable

View File

@ -0,0 +1,77 @@
/**
* @file HashBucket.cc
* @brief HashBucket
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "HashBucket.h"
namespace LightTable {
HashBucket::HashBucket() : bucketSize(0) {}
uint32_t HashBucket::addBucketItem(uint64_t key, RowLocation rowLocation) {
std::pair<std::map<uint64_t, RowLocation>::iterator, bool> ret;
// std::unique_lock<std::mutex> lck(mtx);
// pthread_mutex_lock(&mtx);
ret = bucketItems.insert(std::pair<uint64_t, RowLocation>(key, rowLocation));
// lck.unlock();
// pthread_mutex_unlock(&mtx);
if (ret.second == false) {
return ADD_BUCKET_ITEM_ERROR;
}
bucketSize++;
return SUCCESS;
}
uint32_t HashBucket::deleteBucketItem(uint64_t key) {
if (bucketItems.erase(key) == 1) {
bucketSize--;
return SUCCESS;
}
return BUCKET_ITEM_NOT_FOUND;
}
uint32_t HashBucket::updateRowLocation(uint64_t key, RowLocation rowLocation) {
std::map<uint64_t, RowLocation>::iterator iter;
iter = bucketItems.find(key);
if (iter != bucketItems.end()) {
(iter->second).blockID = rowLocation.blockID;
(iter->second).rowOffset = rowLocation.rowOffset;
return SUCCESS;
}
return BUCKET_ITEM_NOT_FOUND;
}
uint32_t HashBucket::getRowLocation(uint64_t key, RowLocation &rowLocation) {
std::map<uint64_t, RowLocation>::iterator iter;
iter = bucketItems.find(key);
if (iter != bucketItems.end()) {
rowLocation.blockID = (iter->second).blockID;
rowLocation.rowOffset = (iter->second).rowOffset;
return SUCCESS;
}
rowLocation.blockID = 0;
rowLocation.rowOffset = 0;
return BUCKET_ITEM_NOT_FOUND;
}
uint64_t HashBucket::getBucketSize() { return bucketSize; }
uint32_t HashBucket::setBucketSize(uint64_t bucketSize) {
this->bucketSize = bucketSize;
return SUCCESS;
}
uint32_t HashBucket::getAllBucketItems(
std::map<uint64_t, RowLocation> &bucketItems) {
bucketItems = this->bucketItems;
return SUCCESS;
}
} // namespace LightTable

View File

@ -0,0 +1,113 @@
/**
* @file HashTable.cc
* @brief HashTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "HashTable.h"
namespace LightTable {
HashTable::HashTable() {
buckets.reserve(HASH_BUCKET_SIZE);
for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) {
HashBucket *tempBucket = new HashBucket();
buckets.push_back(tempBucket);
}
}
HashTable::~HashTable() {
std::vector<HashBucket *>::iterator iter = buckets.begin();
while (iter != buckets.end()) {
iter = buckets.erase(iter);
}
}
uint64_t HashTable::getMapKey(const char *primaryKey) {
const std::string str = primaryKey;
uint64_t BitsInUnsignedInt = (uint64_t)(4 * 8);
uint64_t ThreeQuarters = (uint64_t)((BitsInUnsignedInt * 3) / 4);
uint64_t OneEighth = (uint64_t)(BitsInUnsignedInt / 8);
uint64_t HighBits = (uint64_t)(0xFFFFFFFF) << (BitsInUnsignedInt - OneEighth);
uint64_t hash = 0;
uint64_t test = 0;
for (uint32_t i = 0; i < str.length(); i++) {
hash = (hash << OneEighth) + str[i];
if ((test = hash & HighBits) != 0) {
hash = ((hash ^ (test >> ThreeQuarters)) & (~HighBits));
}
}
return hash;
}
uint32_t HashTable::hashToBucket(uint64_t key) {
return key % HASH_BUCKET_SIZE;
}
uint32_t HashTable::getRowLocation(uint32_t bucketID, uint64_t key,
RowLocation &rowLocation) {
return buckets[bucketID]->getRowLocation(key, rowLocation);
}
uint32_t HashTable::addItem(uint32_t bucketID, uint64_t key,
RowLocation rowLocation) {
return buckets[bucketID]->addBucketItem(key, rowLocation);
}
uint32_t HashTable::deleteItem(uint32_t bucketID, uint64_t key) {
return buckets[bucketID]->deleteBucketItem(key);
}
uint32_t HashTable::updateItem(uint32_t bucketID, uint64_t key,
RowLocation rowLocation) {
return buckets[bucketID]->updateRowLocation(key, rowLocation);
}
uint64_t HashTable::getItemCount() {
if (buckets.size() == 0) {
return buckets.size();
}
uint64_t itemCount = 0;
for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) {
itemCount += buckets[i]->getBucketSize();
}
return itemCount;
}
uint32_t HashTable::getAllBuckets(std::vector<HashBucket *> &buckets) {
buckets = this->buckets;
return SUCCESS;
}
uint32_t HashTable::refreshRowLocations(
uint64_t blockID, uint64_t deleteOffset, uint64_t rowSize,
std::map<uint64_t, RowLocation> &locationItems) {
for (uint32_t i = 0; i < HASH_BUCKET_SIZE; i++) {
std::map<uint64_t, RowLocation> bucketItems;
buckets[i]->getAllBucketItems(bucketItems);
std::map<uint64_t, RowLocation>::iterator iter = bucketItems.begin();
while (iter != bucketItems.end()) {
if ((iter->second).blockID == blockID &&
(iter->second).rowOffset > deleteOffset) {
(iter->second).rowOffset -= rowSize;
locationItems.insert(
std::pair<uint64_t, RowLocation>(iter->first, iter->second));
}
}
}
return SUCCESS;
}
uint32_t HashTable::clear() {
std::vector<HashBucket *>::iterator iter = buckets.begin();
while (iter != buckets.end()) {
iter = buckets.erase(iter);
}
return SUCCESS;
}
} // namespace LightTable

View File

@ -0,0 +1,23 @@
ifndef LIGHTTABLE_DIR
LIGHTTABLE_DIR = ../..
endif
SRC_FILES := $(wildcard *.cc)
INCLUDE_DIR = -I$(LIGHTTABLE_DIR)/src -I. -I$(LIGHTTABLE_DIR)/spdk/include -I/opt/gnu-mcu-eclipse/riscv-none-gcc/8.2.0-2.1-20190425-1021/riscv-none-embed/include/c++/8.2.0
SOURCE_FOR_CC = $(foreach source_file_1, $(SOURCES),$(filter %.cc, $(source_file_1)))
CXXFLAGS += -fno-omit-frame-pointer \
-Wno-missing-field-initializers -Wmissing-declarations -fno-strict-aliasing \
$(INCLUDE_DIR) -Wformat -Wformat-security -D_GNU_SOURCE \
-fPIC -fno-stack-protector -fno-common -DNDEBUG -U_FORTIFY_SOURCE \
-D_FORTIFY_SOURCE=2 -std=c++11
CXXFLAGS += -DYCSB_TEST
CXXFLAGS += -Wl,-z,relro,-z,now -Wl,-z,noexecstack -lpthread -Wl,-z,relro,-z,now \
-Wl,-z,noexecstack \
-Wl,--whole-archive -Wl,--no-whole-archive -lrt
include $(KERNEL_ROOT)/compiler.mk

View File

@ -0,0 +1,267 @@
/**
* @file PrefetchBlockManager.cc
* @brief PrefetchBlockManager
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "PrefetchBlockManager.h"
namespace LightTable
{
PrefetchBlockManager::PrefetchBlockManager(uint64_t firstBlockID,
uint64_t segmentID,
SegmentType segmentType)
: segmentID(segmentID), segmentType(segmentType), firstBlockID(firstBlockID), currentID(0)
{
uint32_t num = 0;
switch (segmentType)
{
case SEGMENT_TYPE_SMALL:
num = SEGMENT_TYPE_SMALL_CELL_SIZE / BLOCK_SIZE;
break;
case SEGMENT_TYPE_MEDIUM:
num = SEGMENT_TYPE_MEDIUM_CELL_SIZE / BLOCK_SIZE;
break;
case SEGMENT_TYPE_BIG:
num = SEGMENT_TYPE_BIG_CELL_SIZE / BLOCK_SIZE;
break;
default:
break;
}
blockCount = num;
#ifndef YCSB_TEST
while (num--)
{
PrefetchBlockManagerEntry *entry = new PrefetchBlockManagerEntry();
entry->blockID = firstBlockID + num;
entry->currentOffset = 0;
entry->isOccupied = PREFETCH_BLOCK_STATUS_IDLE;
prefetchBlocks.insert(std::pair<uint64_t, PrefetchBlockManagerEntry>(entry->blockID, *entry));
}
#endif
}
PrefetchBlockManager::~PrefetchBlockManager()
{
prefetchBlocks.clear();
}
uint64_t
PrefetchBlockManager::getNextBlock()
{
uint32_t count = blockCount - currentID;
if (count <= 0)
{
return 0;
}
currentID++;
return currentID + firstBlockID - 1;
}
uint64_t
PrefetchBlockManager::blockAllocate(uint64_t rowSize)
{
if (blockCount == 0)
{
return 0;
}
// struct timespec time;
// clock_gettime(CLOCK_REALTIME, &time);
srand((unsigned)time(NULL));
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
uint32_t count = blockCount - currentID;
while (count--)
{
uint64_t blockID = currentID + firstBlockID;
iter = prefetchBlocks.find(blockID);
if (!iter->second.isOccupied)
{
if (rowSize < (BLOCK_SIZE - iter->second.currentOffset))
{
iter->second.isOccupied = PREFETCH_BLOCK_STATUS_BUSY;
return blockID;
}
}
currentID++;
}
return 0;
}
uint32_t
PrefetchBlockManager::setBlockState(uint64_t blockID, bool isOccupied)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
iter->second.isOccupied = isOccupied;
return SUCCESS;
}
else
{
return PREFETCH_BLOCK_NOT_FOUND;
}
}
uint32_t
PrefetchBlockManager::setOffset(uint64_t blockID, uint64_t currentOffset)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
iter->second.currentOffset = currentOffset;
return SUCCESS;
}
else
{
return PREFETCH_BLOCK_NOT_FOUND;
}
}
uint32_t
PrefetchBlockManager::getOffset(uint64_t blockID, uint64_t &currentOffset)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
currentOffset = iter->second.currentOffset;
return SUCCESS;
}
else
{
return PREFETCH_BLOCK_NOT_FOUND;
}
}
uint32_t
PrefetchBlockManager::advanceOffset(uint64_t blockID, uint64_t rowSize)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
iter->second.currentOffset += rowSize;
return SUCCESS;
}
else
{
return PREFETCH_BLOCK_NOT_FOUND;
}
}
uint32_t
PrefetchBlockManager::isOccupied(uint64_t blockID, bool &status)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
status = iter->second.isOccupied;
return SUCCESS;
}
else
{
return PREFETCH_BLOCK_NOT_FOUND;
}
}
uint32_t
PrefetchBlockManager::getPrefetchBlockEntry(uint64_t blockID,
PrefetchBlockManagerEntry &entry)
{
entry = prefetchBlocks[blockID];
return SUCCESS;
}
uint32_t
PrefetchBlockManager::getAllPrefetchBlockEntrys(std::map<uint64_t,
PrefetchBlockManagerEntry> &entrys)
{
entrys = prefetchBlocks;
return SUCCESS;
}
uint32_t
PrefetchBlockManager::lockBlock(uint64_t blockID)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
if (iter->second.isOccupied == PREFETCH_BLOCK_STATUS_BUSY)
{
return BLOCK_OCCUPIED;
}
iter->second.isOccupied = PREFETCH_BLOCK_STATUS_BUSY;
return SUCCESS;
}
return PREFETCH_BLOCK_NOT_FOUND;
}
uint32_t
PrefetchBlockManager::unlockBlock(uint64_t blockID)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
if (iter->second.isOccupied == PREFETCH_BLOCK_STATUS_BUSY)
{
iter->second.isOccupied = PREFETCH_BLOCK_STATUS_IDLE;
}
return SUCCESS;
}
return PREFETCH_BLOCK_NOT_FOUND;
}
bool
PrefetchBlockManager::isCached(uint64_t blockID, uint64_t &pageID)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end() && iter->second.isCached)
{
pageID = iter->second.pageID;
return true;
}
return false;
}
uint32_t
PrefetchBlockManager::setIsCached(uint64_t blockID, bool isCached, uint64_t pageID)
{
std::map<uint64_t, PrefetchBlockManagerEntry>::iterator iter;
iter = prefetchBlocks.find(blockID);
if (iter != prefetchBlocks.end())
{
iter->second.isCached = isCached;
if (isCached)
{
iter->second.pageID = pageID;
}
return SUCCESS;
}
return PREFETCH_BLOCK_NOT_FOUND;
}
} // namespace LightTable

View File

@ -0,0 +1,244 @@
/**
* @file RootTable.cc
* @brief RootTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "RootTable.h"
namespace LightTable {
RootTable::RootTable(bool usePrimaryKeyIndex)
: SystemTable(ROOTTABLE_TABLE_ID, "RootTable", ROOTTABLE_FIRST_BLOCK_ID,
ROOTTABLE_SEGMENT_ID, SEGMENT_TYPE_SMALL,
usePrimaryKeyIndex) {
Schema::SchemaEntry entry1 = {1, "tableID", "uint64_t", 8, true, false};
schema.appendEntry(entry1);
Schema::SchemaEntry entry2 = {2, "tableName", "string", TABLE_NAME_LENGTH,
false, false};
schema.appendEntry(entry2);
Schema::SchemaEntry entry3 = {3, "firstBlockID", "uint64_t", 8, false, false};
schema.appendEntry(entry3);
Schema::SchemaEntry entry4 = {4, "segmentID", "uint64_t", 8, false, false};
schema.appendEntry(entry4);
Schema::SchemaEntry entry5 = {5, "segmentType", "uint32_t", 4, false, false};
schema.appendEntry(entry5);
Schema::SchemaEntry entry6 = {6, "user", "string", USER_NAME_LENGTH,
false, true};
schema.appendEntry(entry6);
Schema::SchemaEntry entry7 = {7, "time", "time", 60, false, false};
schema.appendEntry(entry7);
}
uint32_t RootTable::tableNameToTableID(const char *tableName,
uint64_t &tableID) {
std::map<uint64_t, TableTuple>::iterator iter = tableTupleMap.begin();
while (iter != tableTupleMap.end()) {
if (0 == strncmp(tableName, iter->second.tableName, TABLE_NAME_LENGTH)) {
tableID = iter->second.tableID;
return SUCCESS;
}
iter++;
}
return TABLE_NOT_FOUND;
}
uint32_t RootTable::deleteTableInfo(uint64_t tableID) {
uint32_t ret = SUCCESS;
RowLocation rowLocation;
uint32_t bucketID = hashTable.hashToBucket(tableID);
ret = hashTable.getRowLocation(bucketID, tableID, rowLocation);
uint64_t deleteBlockID = rowLocation.blockID;
uint64_t deleteRowOffset = rowLocation.rowOffset;
char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH];
uint32_t size = 0;
while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS)
;
uint64_t currentOffset = 0;
ret = prefetchBlockManager.getOffset(rowLocation.blockID, currentOffset);
if (ret != SUCCESS) {
return ret;
}
if (rowLocation.rowOffset + schema.getEntrySize() != currentOffset) {
// RowBufferItem::Tuple item;
// uint8_t str[ROOTTABLE_TUPLE_SIZE];
// memcpy(str, &tableTupleMap[tableID], sizeof(tableTupleMap[tableID]));
// item.blockID = rowLocation.blockID;
// item.rowOffset = rowLocation.rowOffset;
// item.isDelete = true;
// RowBufferItem *rowBufferItem = new RowBufferItem(item);
// buf->append(rowBufferItem);
std::map<uint64_t, RowLocation> locationItems;
hashTable.refreshRowLocations(deleteBlockID, deleteRowOffset,
schema.getEntrySize(), locationItems);
// std::map<uint64_t, RowLocation>::iterator iter = locationItems.begin();
// while (iter != locationItems.end()) {
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(iter->first,
// iter->second), tableID, hashTableMetaData, size);
// MetaBufferItem *nextMeta= new MetaBufferItem(tableID, hashTableMetaData,
// size); buf->append(nextMeta);
// }
}
hashTable.deleteItem(bucketID, tableID);
rowLocation.blockID = 0;
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(tableID,
// rowLocation),
// ROOTTABLE_TABLE_ID, hashTableMetaData, size);
// MetaBufferItem *hashTableMeta= new MetaBufferItem(ROOTTABLE_TABLE_ID,
// hashTableMetaData, size);
// buf->append(hashTableMeta);
uint64_t newOffset = currentOffset - schema.getEntrySize();
prefetchBlockManager.setOffset(deleteBlockID, newOffset);
PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry;
prefetchBlockManager.unlockBlock(deleteBlockID);
prefetchBlockManager.getPrefetchBlockEntry(deleteBlockID, blockEntry);
// char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH];
// MetaHandle::serialize(blockEntry, ROOTTABLE_TABLE_ID, pbmData, size);
// MetaBufferItem *pbmMeta = new MetaBufferItem(ROOTTABLE_TABLE_ID, pbmData,
// size); buf->append(pbmMeta);
if (tableTupleMap.erase(tableID) == 1) {
return SUCCESS;
}
return ERROR_TABLE;
}
uint32_t RootTable::appendTableInfo(TableTuple tableTuple) {
uint32_t err = SUCCESS;
tableTupleMap.insert(
std::pair<uint64_t, TableTuple>(tableTuple.tableID, tableTuple));
uint64_t blockID = 0;
uint64_t blockOffset = 0;
blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize());
err = prefetchBlockManager.getOffset(blockID, blockOffset);
if (err != SUCCESS) {
// TODO allocate new seg
return GET_AVAILABLE_BLOCK_ERROR;
}
// RowBufferItem::Tuple item;
// uint8_t str[ROOTTABLE_TUPLE_SIZE];
// memcpy(str, &tableTuple, sizeof(tableTuple));
// item.row = str;
// item.blockID = blockID;
// item.rowOffset = blockOffset;
// item.isDelete = false;
// RowBufferItem *rowBufferItem = new RowBufferItem(item);
// buf->append(rowBufferItem);
uint32_t bucketID = hashTable.hashToBucket(tableTuple.tableID);
RowLocation rowLocation = {blockID, blockOffset};
err = hashTable.addItem(bucketID, tableTuple.tableID, rowLocation);
if (err != SUCCESS) {
return err;
}
// uint32_t size = 0;
// char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH];
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(tableTuple.tableID,
// rowLocation), tableID, hashTableMetaData, size);
// MetaBufferItem *hashTableMeta = new MetaBufferItem(tableID,
// hashTableMetaData,
// size);
// buf->append(hashTableMeta);
err = prefetchBlockManager.advanceOffset(blockID, schema.getEntrySize());
prefetchBlockManager.setBlockState(blockID, PREFETCH_BLOCK_STATUS_IDLE);
// PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry;
// prefetchBlockManager.getPrefetchBlockEntry(blockID, blockEntry);
// char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH];
// MetaHandle::serialize(blockEntry, tableID, pbmData, size);
// MetaBufferItem *pbmMeta = new MetaBufferItem(tableID, pbmData, size);
// buf->append(pbmMeta);
return SUCCESS;
}
uint32_t RootTable::getTableInfo(uint64_t tableID,
TableTuple &tableTupleResult) {
std::map<uint64_t, TableTuple>::iterator iter;
iter = tableTupleMap.find(tableID);
if (iter != tableTupleMap.end()) {
tableTupleResult = iter->second;
return SUCCESS;
}
tableTupleResult.tableID = 0;
return TABLE_TUPLE_NOT_FOUND;
}
uint32_t RootTable::modifyTableInfo(uint64_t tableID, TableTuple tableTuple) {
uint32_t ret = SUCCESS;
std::map<uint64_t, TableTuple>::iterator iter;
iter = tableTupleMap.find(tableID);
if (iter != tableTupleMap.end()) {
iter->second = tableTuple;
uint32_t bucketID = hashTable.hashToBucket(tableID);
RowLocation rowLocation;
ret = hashTable.getRowLocation(bucketID, tableID, rowLocation);
if (ret != SUCCESS) {
return ret;
}
// RowBufferItem::Tuple item;
// uint8_t str[ROOTTABLE_TUPLE_SIZE];
// memcpy(str, &tableTuple, sizeof(tableTuple));
// item.row = str;
// item.blockID = rowLocation.blockID;
// item.rowOffset = rowLocation.rowOffset;
// item.isDelete = false;
// RowBufferItem *rowBufferItem = new RowBufferItem(item);
// buf->append(rowBufferItem);
return SUCCESS;
}
return TABLE_TUPLE_NOT_FOUND;
}
uint32_t RootTable::getTableTuple(uint64_t tableID, TableTuple &tableTuple) {
std::map<uint64_t, TableTuple>::iterator iter;
iter = tableTupleMap.find(tableID);
if (iter != tableTupleMap.end()) {
tableTuple = iter->second;
return SUCCESS;
}
return TABLE_TUPLE_NOT_FOUND;
}
uint32_t RootTable::serializeRow(const TableTuple tableTuple, uint8_t *buf) {
memcpy(buf, &tableTuple, ROOTTABLE_TUPLE_SIZE);
return SUCCESS;
}
uint32_t RootTable::deserializeRow(const uint8_t *buf,
TableTuple *tableTupleResult) {
if (buf != NULL) {
memcpy(tableTupleResult, buf, ROOTTABLE_TUPLE_SIZE);
if (tableTupleResult->tableID != 0) {
return SUCCESS;
}
}
return BUFFER_EMPTY;
}
uint64_t RootTable::getTableCount() { return tableTupleMap.size(); }
} // namespace LightTable

View File

@ -0,0 +1,77 @@
/**
* @file RowMap.cc
* @brief RowMap
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "RowMap.h"
namespace LightTable {
RowMap::RowMap() {}
uint32_t RowMap::appendEntry(uint64_t key, uint64_t blockID,
uint64_t rowOffset) {
std::pair<std::map<uint64_t, RowMapEntry>::iterator, bool> ret;
RowMapEntry rowMapEntry = {key, blockID, rowOffset};
ret = rowMapEntrys.insert(std::pair<uint64_t, RowMapEntry>(key, rowMapEntry));
if (ret.second == false) {
return ADD_ROW_MAP_ENTRY_ERROR;
}
return SUCCESS;
}
uint32_t RowMap::deleteEntry(uint64_t key) {
if (rowMapEntrys.erase(key) == 1) {
return SUCCESS;
}
return ROW_MAP_ENTRY_NOT_FOUND;
}
uint32_t RowMap::alterEntry(uint64_t key, uint64_t blockID,
uint64_t rowOffset) {
std::map<uint64_t, RowMapEntry>::iterator iter;
iter = rowMapEntrys.find(key);
if (iter != rowMapEntrys.end()) {
(iter->second).key = key;
(iter->second).blockID = blockID;
(iter->second).rowOffset = rowOffset;
return SUCCESS;
}
return ROW_MAP_ENTRY_NOT_FOUND;
}
uint32_t RowMap::queryEntry(uint64_t key, RowMapEntry &rowMapEntry) {
std::map<uint64_t, RowMapEntry>::iterator iter;
iter = rowMapEntrys.find(key);
if (iter != rowMapEntrys.end()) {
rowMapEntry = iter->second;
return SUCCESS;
}
rowMapEntry.key = key;
rowMapEntry.blockID = 0;
rowMapEntry.rowOffset = 0;
return ROW_MAP_ENTRY_NOT_FOUND;
}
uint32_t RowMap::queryAllEntry(
std::map<uint64_t, RowMapEntry> &allRowMapEntrys) {
if (rowMapEntrys.empty() == false) {
allRowMapEntrys = rowMapEntrys;
return SUCCESS;
}
return ROW_MAP_ENTRY_NOT_FOUND;
}
uint64_t RowMap::getRowCount() { return rowMapEntrys.size(); }
} // namespace LightTable

View File

@ -0,0 +1,166 @@
/**
* @file Schema.cc
* @brief table schema
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "Schema.h"
namespace LightTable {
Schema::Schema() {
primaryKeySchema.columnID = 1;
primaryKeySchema.isPrimaryKey = true;
columnCount = 0;
entrySize = 0;
}
Schema::Schema(std::vector<SchemaEntry> schemaEntrys) {
columnCount = 0;
entrySize = 0;
primaryKeySchema.columnID = 1;
primaryKeySchema.isPrimaryKey = true;
std::vector<SchemaEntry>::iterator iter = schemaEntrys.begin();
while (iter != schemaEntrys.end()) {
appendEntry(*iter);
iter++;
}
}
uint32_t Schema::appendEntry(SchemaEntry &schemaEntry) {
if (schemaEntry.columnID != columnCount + 1) {
return COLUMN_ID_NOT_CONTINUOUS;
}
std::pair<std::map<uint64_t, SchemaEntry>::iterator, bool> ret;
ret = schemaEntrys.insert(std::pair<uint64_t, Schema::SchemaEntry>(
schemaEntry.columnID, schemaEntry));
if (ret.second == false) {
return ADD_SCHEMA_ENTRY_ERROR;
}
if (schemaEntry.isPrimaryKey) {
primaryKeySchema = schemaEntry;
}
columnCount++;
entrySize = entrySize + schemaEntry.length;
return SUCCESS;
}
uint32_t Schema::deleteEntry(uint64_t columnID) {
if (columnID == primaryKeySchema.columnID) {
return DELETE_PARIMARY_KEY_COLUMN;
}
uint64_t newColumnID = columnID;
SchemaEntry schemaEntry;
std::map<uint64_t, SchemaEntry>::iterator iter;
iter = schemaEntrys.find(columnID);
if (iter != schemaEntrys.end()) {
schemaEntry = iter->second;
schemaEntrys.erase(iter);
entrySize = entrySize - schemaEntry.length;
columnCount--;
if (columnCount > 0) {
iter = schemaEntrys.find(newColumnID + 1);
while (iter != schemaEntrys.end()) {
schemaEntry = iter->second;
schemaEntry.columnID = newColumnID;
schemaEntrys.insert(
std::pair<uint64_t, Schema::SchemaEntry>(newColumnID, schemaEntry));
schemaEntrys.erase(iter);
newColumnID++;
iter = schemaEntrys.find(newColumnID + 1);
}
}
return SUCCESS;
}
return SCHEMA_ENTRY_NOT_FOUND;
}
uint32_t Schema::alterEntry(uint64_t columnID, SchemaEntry schemaEntry) {
if (schemaEntry.columnID != columnID) {
return COLUMN_ID_NOT_CONTINUOUS;
}
std::map<uint64_t, SchemaEntry>::iterator iter;
iter = schemaEntrys.find(columnID);
entrySize = entrySize - (iter->second).length;
if (iter != schemaEntrys.end()) {
iter->second = schemaEntry;
primaryKeySchema = schemaEntry;
entrySize = entrySize + (iter->second).length;
return SUCCESS;
}
return SCHEMA_ENTRY_NOT_FOUND;
}
uint32_t Schema::getColumnID(const char *columnName, uint64_t &columnID) {
std::map<uint64_t, SchemaEntry>::iterator iter = schemaEntrys.begin();
while (iter != schemaEntrys.end()) {
if (strncmp(iter->second.columnName, columnName, iter->second.length) ==
0) {
columnID = (iter->second).columnID;
return SUCCESS;
}
iter++;
}
return COLUMN_ITEM_NOT_FOUND;
}
uint32_t Schema::queryEntry(uint64_t columnID,
Schema::SchemaEntry &schemaEntry) {
std::map<uint64_t, SchemaEntry>::iterator iter;
iter = schemaEntrys.find(columnID);
if (iter != schemaEntrys.end()) {
schemaEntry = iter->second;
return SUCCESS;
}
schemaEntry.columnID = 0;
return SCHEMA_ENTRY_NOT_FOUND;
}
uint32_t Schema::queryAllEntry(
std::map<uint64_t, SchemaEntry> &allSchemaEntrys) {
allSchemaEntrys = schemaEntrys;
if (schemaEntrys.empty() == false) {
return SUCCESS;
}
return SCHEMA_ENTRY_NOT_FOUND;
}
uint64_t Schema::getEntrySize() { return entrySize; }
void Schema::setEntrySize(uint64_t entrySize) { this->entrySize = entrySize; }
uint64_t Schema::getColumnCount() { return columnCount; }
void Schema::setColumnCount(uint64_t columnCount) {
this->columnCount = columnCount;
}
uint32_t Schema::getPrimaryKeyLength() { return primaryKeySchema.length; }
Schema::SchemaEntry &Schema::getPrimaryKeySchema() { return primaryKeySchema; }
uint32_t Schema::drop() {
schemaEntrys.clear();
primaryKeySchema.length = 0;
columnCount = 0;
entrySize = 0;
return SUCCESS;
}
} // namespace LightTable

View File

@ -0,0 +1,180 @@
/**
* @file SegmentManager.cc
* @brief SegmentManager
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "SegmentManager.h"
namespace LightTable {
SegmentManager::SegmentManager() {
int blockID, num = 0;
smallBitmap = new uint8_t[SEGMENT_TYPE_SMALL_BITMAP_SIZE];
mediumBitmap = new uint8_t[SEGMENT_TYPE_MEDIUM_BITMAP_SIZE];
bigBitmap = new uint8_t[SEGMENT_TYPE_BIG_BITMAP_SIZE];
#ifdef YCSB_TEST
blockID = SEGMENT_TYPE_BIG_START;
while (num < SEGMENT_TYPE_BIG_BITMAP_SIZE) {
Driver::read(bigBitmap + num, blockID);
num += BLOCK_SIZE;
blockID++;
}
#else
num = 0;
blockID = SEGMENT_TYPE_SMALL_START;
while (num < SEGMENT_TYPE_SMALL_BITMAP_SIZE) {
Driver::read(smallBitmap + num, blockID);
num += BLOCK_SIZE;
blockID++;
}
num = 0;
blockID = SEGMENT_TYPE_MEDIUM_START;
while (num < SEGMENT_TYPE_MEDIUM_BITMAP_SIZE) {
Driver::read(mediumBitmap + num, blockID);
num += BLOCK_SIZE;
blockID++;
}
num = 0;
blockID = SEGMENT_TYPE_BIG_START;
while (num < SEGMENT_TYPE_BIG_BITMAP_SIZE) {
Driver::read(mediumBitmap + num, blockID);
num += BLOCK_SIZE;
blockID++;
}
#endif
}
SegmentManager::~SegmentManager() {
delete[] smallBitmap;
delete[] mediumBitmap;
delete[] bigBitmap;
}
uint32_t SegmentManager::setBitmap(SegmentType segmentType, uint64_t segmentID,
SegmentStatus segmentStatus) {
switch (segmentType) {
case SEGMENT_TYPE_SMALL:
return setBit(segmentID, smallBitmap, SEGMENT_TYPE_SMALL_START,
segmentStatus);
case SEGMENT_TYPE_MEDIUM:
return setBit(segmentID, mediumBitmap, SEGMENT_TYPE_MEDIUM_START,
segmentStatus);
case SEGMENT_TYPE_BIG:
return setBit(segmentID, bigBitmap, SEGMENT_TYPE_BIG_START,
segmentStatus);
default:
return SEGMENT_TYPE_ERROR;
}
}
uint32_t SegmentManager::getIdleSegment(SegmentType segmentType,
uint64_t &segmentID,
uint64_t &firstBlockID) {
uint32_t stateCode;
switch (segmentType) {
case SEGMENT_TYPE_SMALL:
stateCode =
findFirstIdle(smallBitmap, SEGMENT_TYPE_SMALL_BITMAP_SIZE, segmentID);
if (stateCode != SUCCESS) return stateCode;
firstBlockID = SEGMENT_TYPE_SMALL_START +
(SEGMENT_TYPE_SMALL_CELL_SIZE / BLOCK_SIZE) * segmentID;
return setBit(segmentID, smallBitmap, SEGMENT_TYPE_SMALL_START,
SEGMENT_STATUS_BUSY);
case SEGMENT_TYPE_MEDIUM:
stateCode = findFirstIdle(mediumBitmap, SEGMENT_TYPE_MEDIUM_BITMAP_SIZE,
segmentID);
if (stateCode != SUCCESS) return stateCode;
firstBlockID = SEGMENT_TYPE_MEDIUM_START +
(SEGMENT_TYPE_MEDIUM_CELL_SIZE / BLOCK_SIZE) * segmentID;
return setBit(segmentID, mediumBitmap, SEGMENT_TYPE_MEDIUM_START,
SEGMENT_STATUS_BUSY);
case SEGMENT_TYPE_BIG:
stateCode =
findFirstIdle(bigBitmap, SEGMENT_TYPE_BIG_BITMAP_SIZE, segmentID);
if (stateCode != SUCCESS) return stateCode;
firstBlockID = SEGMENT_TYPE_BIG_START +
(SEGMENT_TYPE_BIG_CELL_SIZE / BLOCK_SIZE) * segmentID;
return setBit(segmentID, bigBitmap, SEGMENT_TYPE_BIG_START,
SEGMENT_STATUS_BUSY);
default:
return SEGMENT_TYPE_ERROR;
}
}
uint32_t SegmentManager::setBit(uint64_t segmentID, uint8_t *buf,
uint64_t segmentStart,
SegmentStatus segmentStatus) {
uint32_t integer, remainder;
uint8_t tmp = 1;
integer = segmentID / 8;
remainder = segmentID % 8;
tmp = tmp << remainder;
if (segmentStatus == SEGMENT_STATUS_BUSY) {
buf[integer] |= tmp;
} else if (segmentStatus == SEGMENT_STATUS_IDLE) {
buf[integer] &= ~tmp;
} else {
return SEGMENT_STATUS_ERROR;
}
uint64_t num = integer / BLOCK_SIZE;
uint64_t blockID = num + segmentStart;
Driver::write(buf + num * BLOCK_SIZE, blockID);
return SUCCESS;
}
uint32_t SegmentManager::findFirstIdle(uint8_t *buf, uint64_t segmentNum,
uint64_t &segmentID) {
uint64_t num, place;
uint8_t res, tmp, isfound;
for (isfound = num = 0; num < segmentNum; num++) {
for (place = 0; place < 8; place++) {
tmp = 0x01 << place;
tmp = tmp & buf[num];
res = (tmp >> place);
if (res == 0) {
isfound = 1;
break;
}
}
if (isfound == 1) break;
}
if (isfound == 0) {
return SEGMENT_IS_EXHAUSTED;
}
segmentID = num * 8 + place;
return SUCCESS;
}
} // namespace LightTable

View File

@ -0,0 +1,31 @@
/**
* @file SystemTable.cc
* @brief SystemTable
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "SystemTable.h"
namespace LightTable {
SystemTable::SystemTable(uint64_t tableID, const char *tableName,
uint64_t firstBlockID, uint64_t segmentID,
SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex)
: Table(tableID, tableName, firstBlockID, segmentID, segmentType,
schemaEntrys, usePrimaryKeyIndex)
{}
SystemTable::SystemTable(uint64_t tableID, const char *tableName,
uint64_t firstBlockID, uint64_t segmentID,
SegmentType segmentType, bool usePrimaryKeyIndex)
: Table(tableID, tableName, firstBlockID, segmentID, segmentType,
usePrimaryKeyIndex)
{}
} // namespace LightTable

View File

@ -0,0 +1,639 @@
/**
* @file Table.cc
* @brief Table
* @version 0.1
* @author SYS Lab
* @date 2022.11.01
*/
#include "Table.h"
namespace LightTable {
Table::Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
std::vector<Schema::SchemaEntry> schemaEntrys,
bool usePrimaryKeyIndex)
: prefetchBlockManager(firstBlockID, segmentID, segmentType),
schema(schemaEntrys),
rowMap(),
tupleNum(0),
t(time(0)),
indexLocation{TABLE1_INDEX_BLOCKID, META_ENTRY_NUM},
metaLogLocation{TABLE1_META_LOG_BLOCKID, META_ENTRY_NUM},
indexLogLocation{TABLE1_INDEX_LOG_BLOCKID, META_ENTRY_NUM},
dataLogLocation{TABLE1_DATA_LOG_BLOCKID, META_ENTRY_NUM} {
this->tableID = tableID;
memcpy(this->tableName, tableName, sizeof(tableName) + 1);
this->usePrimaryKeyIndex = usePrimaryKeyIndex;
std::map<uint64_t, Schema::SchemaEntry> allSchemaEntrys;
schema.queryAllEntry(allSchemaEntrys);
}
Table::Table(uint64_t tableID, const char *tableName, uint64_t firstBlockID,
uint64_t segmentID, SegmentType segmentType,
bool usePrimaryKeyIndex)
: prefetchBlockManager(firstBlockID, segmentID, segmentType),
schema(),
rowMap() {
this->tableID = tableID;
memcpy(this->tableName, tableName, sizeof(tableName) + 1);
// this->buf = buf;
// this->queryCache = queryCache;
// this->blockPreload = blockPreload;
this->usePrimaryKeyIndex = usePrimaryKeyIndex;
}
uint32_t Table::drop() {
uint32_t ret = SUCCESS;
ret = truncate();
if (ret != SUCCESS) {
return ret;
}
schema.drop();
// std::map<uint64_t, Schema::SchemaEntry> allSchemaEntrys;
// schema.queryAllEntry(allSchemaEntrys);
// uint64_t columnCount = allSchemaEntrys.size();
// uint64_t length = 10 + columnCount * (SCHEMA_ENTRY_SERIALIZED_LENGTH + 1);
// char *schemaData = new char[length];
// uint32_t size = 0;
// MetaHandle::serialize(allSchemaEntrys, tableID, schemaData, size);
// MetaBufferItem *schemaMetaItem = new MetaBufferItem(tableID, schemaData,
// size); buf->append(schemaMetaItem); delete[] schemaData;
return SUCCESS;
}
// uint32_t Table::truncate() {
// uint32_t size = 0;
// char *pbmData;
// MetaBufferItem *pbmMeta;
// char *hashTableData;
// MetaBufferItem *hashTableMeta;
// std::map<uint64_t, PrefetchBlockManager::PrefetchBlockManagerEntry> entrys;
// prefetchBlockManager.getAllPrefetchBlockEntrys(entrys);
// std::map<uint64_t, PrefetchBlockManager::PrefetchBlockManagerEntry>::iterator
// pbmIter = entrys.begin();
// while (pbmIter != entrys.end()) {
// prefetchBlockManager.lockBlock(pbmIter->first);
// prefetchBlockManager.setOffset(pbmIter->first, 0);
// prefetchBlockManager.unlockBlock(pbmIter->first);
// pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH];
// MetaHandle::serialize(pbmIter->second, tableID, pbmData, size);
// pbmMeta = new MetaBufferItem(tableID, pbmData, size);
// // buf->append(pbmMeta);
// pbmIter++;
// }
// std::vector<HashBucket *> buckets;
// hashTable.getAllBuckets(buckets);
// for (uint32_t i = 0; i < buckets.size(); i++) {
// std::map<uint64_t, RowLocation> bucketItems;
// buckets[i]->getAllBucketItems(bucketItems);
// std::map<uint64_t, RowLocation>::iterator itemIter;
// itemIter = bucketItems.begin();
// while (itemIter != bucketItems.end()) {
// itemIter->second.blockID = 0;
// hashTableData = new char[ROWMAP_SERIALIZED_LENGTH];
// MetaHandle::serialize(
// std::pair<uint64_t, RowLocation>(itemIter->first, itemIter->second),
// tableID, hashTableData, size);
// hashTableMeta = new MetaBufferItem(tableID, hashTableData, size);
// // buf->append(hashTableMeta);
// itemIter = bucketItems.erase(itemIter);
// itemIter++;
// }
// }
// return SUCCESS;
// }
uint32_t Table::parsePrimaryKey(const uint8_t *rowData, char *primaryKeyData) {
if (strncmp(schema.getPrimaryKeySchema().type, "uint64_t", 8) == 0) {
uint64_t primaryKeyI64 = 0;
memcpy(&primaryKeyI64, rowData, schema.getPrimaryKeySchema().length);
sprintf(primaryKeyData, "%ld", primaryKeyI64);
} else if (strncmp(schema.getPrimaryKeySchema().type, "string", 6) == 0) {
memcpy(primaryKeyData, rowData, schema.getPrimaryKeySchema().length);
} else {
return TYPE_INVALID;
}
return SUCCESS;
}
uint32_t Table::insertRow(const uint8_t *rowData) {
uint32_t ret = SUCCESS;
uint64_t blockID = 0;
uint64_t blockOffset = 0;
uint8_t blockData[BLOCK_SIZE];
tupleNum++;
t = time(0);
#ifndef YCSB_TEST
blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize());
while (blockID == 0) {
blockID = prefetchBlockManager.blockAllocate(schema.getEntrySize());
}
ret = prefetchBlockManager.getOffset(blockID, blockOffset);
if (ret != SUCCESS) {
// TODO allocate new seg
return ret;
}
#else
blockID = prefetchBlockManager.getNextBlock();
#endif
if (blockOffset != 0) {
Driver::read(blockData, blockID);
}
memcpy(blockData + blockOffset, rowData, schema.getEntrySize());
#ifdef SDCARD_TEST
start_time = std::chrono::steady_clock::now();
#endif
Driver::write(blockData, blockID);
#ifdef SDCARD_TEST
for (int i = 0; i < SDCARD_TEST_NUM; ++i)
Driver::write(blockData, SDCARD_TEST_WRITE_BLOCKID + 100 * i);
end_time = std::chrono::steady_clock::now();
diff_sdcard_write = std::chrono::duration_cast<std::chrono::microseconds>(
end_time - start_time)
.count();
#endif
char *primaryKeyData = new char[schema.getPrimaryKeySchema().length];
ret = parsePrimaryKey(rowData, primaryKeyData);
if (ret != SUCCESS) {
delete[] primaryKeyData;
prefetchBlockManager.unlockBlock(blockID);
return ret;
}
uint64_t key = hashTable.getMapKey(primaryKeyData);
uint32_t bucketID = hashTable.hashToBucket(key);
RowLocation rowLocation = {blockID, blockOffset};
ret = hashTable.addItem(bucketID, key, rowLocation);
if (ret != SUCCESS) {
delete[] primaryKeyData;
prefetchBlockManager.unlockBlock(blockID);
return ret;
}
#ifdef TABLE_STORAGE_SECOND_INDEX
ret = secondaryIndex.insert(primaryKeyData,
schema.getPrimaryKeySchema().length, rowLocation);
if (ret != SUCCESS) {
delete[] primaryKeyData;
prefetchBlockManager.unlockBlock(blockID);
return ret;
}
#endif
#ifndef YCSB_TEST
ret = prefetchBlockManager.advanceOffset(blockID, schema.getEntrySize());
prefetchBlockManager.setBlockState(blockID, PREFETCH_BLOCK_STATUS_IDLE);
#endif
delete[] primaryKeyData;
return ret;
}
uint32_t Table::insertRow_logging(const uint8_t *rowData) {
uint32_t ret = SUCCESS;
uint64_t blockID = 0;
uint64_t blockOffset = 0;
tupleNum++;
t = time(0);
char *primaryKeyData = new char[schema.getPrimaryKeySchema().length];
ret = parsePrimaryKey(rowData, primaryKeyData);
if (ret != SUCCESS) {
delete[] primaryKeyData;
prefetchBlockManager.unlockBlock(blockID);
return ret;
}
uint64_t key = hashTable.getMapKey(primaryKeyData);
uint32_t bucketID = hashTable.hashToBucket(key);
RowLocation rowLocation = {blockID, blockOffset};
ret = hashTable.addItem(bucketID, key, rowLocation);
if (ret != SUCCESS) {
delete[] primaryKeyData;
prefetchBlockManager.unlockBlock(blockID);
return ret;
}
metaLogEntry metaEntry{.tableID = tableID, .tupleNum = tupleNum, .t = t};
if (metaLogLocation.emptySlotNum != 0) {
memcpy(metaLog + (META_ENTRY_SIZE *
(META_ENTRY_NUM - metaLogLocation.emptySlotNum)),
&metaEntry, sizeof(metaLogEntry));
} else {
metaLogLocation.blockID++;
metaLogLocation.emptySlotNum = META_ENTRY_NUM;
memset(metaLog, 0, BLOCK_SIZE);
memcpy(metaLog, &metaEntry, sizeof(metaLogEntry));
}
Driver::write(metaLog, metaLogLocation.blockID);
metaLogLocation.emptySlotNum--;
uint32_t size = 0;
char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH];
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(key, rowLocation),
// tableID, hashTableMetaData, size);
// MetaBufferItem *hashTableMeta =
// new MetaBufferItem(tableID, hashTableMetaData, size);
// if (indexLogLocation.emptySlotNum != 0) {
// memcpy(indexLog + (META_ENTRY_SIZE *
// (META_ENTRY_NUM - indexLogLocation.emptySlotNum)),
// hashTableMeta->metaItem,
// META_ENTRY_SIZE > hashTableMeta->metaItemSize
// ? hashTableMeta->metaItemSize
// : META_ENTRY_SIZE);
// } else {
// indexLogLocation.blockID++;
// indexLogLocation.emptySlotNum = META_ENTRY_NUM;
// memset(indexLog, 0, BLOCK_SIZE);
// memcpy(indexLog, hashTableMeta->metaItem,
// META_ENTRY_SIZE > hashTableMeta->metaItemSize
// ? hashTableMeta->metaItemSize
// : META_ENTRY_SIZE);
// }
Driver::write(indexLog, indexLogLocation.blockID);
indexLogLocation.emptySlotNum--;
memcpy(metaData, &tupleNum, 8);
memcpy(metaData + 8, &t, sizeof(std::time_t));
Driver::write(metaData, TABLE1_META_BLOCKID);
// if (indexLocation.emptySlotNum != 0) {
// memcpy(indexData + (META_ENTRY_SIZE *
// (META_ENTRY_NUM - indexLocation.emptySlotNum)),
// hashTableMeta->metaItem,
// META_ENTRY_SIZE > hashTableMeta->metaItemSize
// ? hashTableMeta->metaItemSize
// : META_ENTRY_SIZE);
// } else {
// indexLocation.blockID++;
// indexLocation.emptySlotNum = META_ENTRY_NUM;
// memset(indexData, 0, BLOCK_SIZE);
// memcpy(indexData, hashTableMeta->metaItem,
// META_ENTRY_SIZE > hashTableMeta->metaItemSize
// ? hashTableMeta->metaItemSize
// : META_ENTRY_SIZE);
// }
Driver::write(indexData, indexLocation.blockID);
indexLocation.emptySlotNum--;
uint8_t blockData[BLOCK_SIZE];
blockID = dataLogLocation.blockID;
memcpy(blockData, rowData, schema.getEntrySize());
Driver::write(blockData, blockID);
dataLogLocation.blockID++;
blockID = prefetchBlockManager.getNextBlock();
if (blockOffset != 0) {
Driver::read(blockData, blockID);
}
memcpy(blockData + blockOffset, rowData, schema.getEntrySize());
Driver::write(blockData, blockID);
delete[] hashTableMetaData;
// delete hashTableMeta;
delete[] primaryKeyData;
return ret;
}
uint32_t Table::deleteRow(const char *primaryKey) {
uint32_t ret = SUCCESS;
RowLocation rowLocation;
ret = locateRow(primaryKey, rowLocation);
if (ret != SUCCESS) {
return ret;
}
uint64_t deleteBlockID = rowLocation.blockID;
uint64_t deleteRowOffset = rowLocation.rowOffset;
while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS)
;
uint64_t currentOffset = 0;
ret = prefetchBlockManager.getOffset(rowLocation.blockID, currentOffset);
uint64_t key = hashTable.getMapKey(primaryKey);
if (rowLocation.rowOffset + schema.getEntrySize() != currentOffset) {
ret = deleteRowData(key, rowLocation);
if (ret != SUCCESS) {
return ret;
}
}
uint32_t size = 0;
uint32_t bucketID = hashTable.hashToBucket(key);
hashTable.deleteItem(bucketID, key);
char *hashTableMetaData = new char[ROWMAP_SERIALIZED_LENGTH];
rowLocation.blockID = 0;
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(key, rowLocation),
// tableID, hashTableMetaData, size);
// MetaBufferItem *hashTableMeta= new MetaBufferItem(tableID,
// hashTableMetaData,
// size);
// buf->append(hashTableMeta);
if (deleteRowOffset + schema.getEntrySize() != currentOffset) {
std::map<uint64_t, RowLocation> locationItems;
hashTable.refreshRowLocations(deleteBlockID, deleteRowOffset,
schema.getEntrySize(), locationItems);
// std::map<uint64_t, RowLocation>::iterator iter = locationItems.begin();
// while (iter != locationItems.end()) {
// MetaHandle::serialize(std::pair<uint64_t, RowLocation>(iter->first,
// iter->second), tableID, hashTableMetaData,
// size);
// MetaBufferItem *nextMeta= new MetaBufferItem(tableID,
// hashTableMetaData,
// size);
// buf->append(nextMeta);
// }
}
uint64_t newOffset = currentOffset - schema.getEntrySize();
prefetchBlockManager.setOffset(deleteBlockID, newOffset);
prefetchBlockManager.unlockBlock(deleteBlockID);
PrefetchBlockManager::PrefetchBlockManagerEntry blockEntry;
prefetchBlockManager.getPrefetchBlockEntry(deleteBlockID, blockEntry);
// char *pbmData = new char[PREFETCH_BLOCK_SERIALIZED_LENGTH];
// MetaHandle::serialize(blockEntry, tableID, pbmData, size);
// MetaBufferItem *pbmMeta = new MetaBufferItem(tableID, pbmData, size);
// buf->append(pbmMeta);
return SUCCESS;
}
uint32_t Table::deleteRowData(uint64_t key, RowLocation rowLocation) {
uint8_t blockData[BLOCK_SIZE];
Driver::read(blockData, rowLocation.blockID);
uint64_t start = rowLocation.rowOffset;
uint64_t nextRow = start + schema.getEntrySize();
memcpy(blockData + start, blockData + nextRow, BLOCK_SIZE - nextRow);
Driver::write(blockData, rowLocation.blockID);
return SUCCESS;
}
uint32_t Table::updateRow(const char *primaryKey, const uint8_t *rowData) {
uint32_t ret = SUCCESS;
RowLocation rowLocation;
ret = locateRow(primaryKey, rowLocation);
if (ret == BUCKET_ITEM_NOT_FOUND) {
return insertRow(rowData);
} else if (ret != SUCCESS) {
return ret;
}
uint8_t blockData[BLOCK_SIZE];
while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS)
;
Driver::read(blockData, rowLocation.blockID);
// char *undoTuple = new char[schema.getEntrySize()];
// memcpy(undoTuple, blockData+rowLocation.rowOffset, schema.getEntrySize());
// UndoLogEntry *logEntry = new UndoLogEntry(this->tableID, primaryKey,
// schema.getPrimaryKeyLength(),
// undoTuple, schema.getEntrySize(),
// rowLocation.blockID, rowLocation.rowOffset);
// buf->append(logEntry);
memcpy(blockData + rowLocation.rowOffset, rowData, schema.getEntrySize());
if (SUCCESS != Driver::write(blockData, rowLocation.blockID)) {
// memcpy(blockData + rowLocation.rowOffset, undoTuple,
// schema.getEntrySize());
// ret = UPDATE_TUPLE_ERROR;
}
// logEntry->expireUndoLog();
prefetchBlockManager.unlockBlock(rowLocation.blockID);
return ret;
}
uint32_t Table::locateRow(const char *primaryKey, RowLocation &rowLocation) {
uint32_t ret = SUCCESS;
uint64_t key = hashTable.getMapKey(primaryKey);
uint32_t bucketID = hashTable.hashToBucket(key);
ret = hashTable.getRowLocation(bucketID, key, rowLocation);
return ret;
}
uint32_t Table::selectRow(const char *primaryKey, uint8_t *rowData) {
RowLocation rowLocation;
return selectRow(primaryKey, rowData, rowLocation);
}
uint32_t Table::selectRow(const char *primaryKey, uint8_t *rowData,
RowLocation &rowLocation) {
uint32_t ret = SUCCESS;
uint8_t blockData[BLOCK_SIZE];
ret = locateRow(primaryKey, rowLocation);
if (ret != SUCCESS) {
return ret;
}
#ifndef YCSB_TEST
while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS)
;
#endif
#ifdef SDCARD_TEST
start_time = std::chrono::steady_clock::now();
#endif
Driver::read(blockData, rowLocation.blockID);
#ifdef SDCARD_TEST
for (int i = 0; i < SDCARD_TEST_NUM; ++i)
Driver::read(blockData, SDCARD_TEST_READ_BLOCKID + 100 * i);
end_time = std::chrono::steady_clock::now();
diff_sdcard_read = (std::chrono::duration_cast<std::chrono::microseconds>(
end_time - start_time)
.count());
#endif
#ifndef YCSB_TEST
prefetchBlockManager.unlockBlock(rowLocation.blockID);
#endif
ret = getRowFromBlock(primaryKey, rowData, blockData, rowLocation);
return ret;
}
uint32_t Table::getRowFromBlock(const char *primaryKey, uint8_t *rowData,
const uint8_t *blockData,
const RowLocation rowLocation) {
uint32_t ret = SUCCESS;
char *primaryKeyData = new char[schema.getPrimaryKeySchema().length];
if (rowLocation.rowOffset + schema.getEntrySize() > BLOCK_SIZE) {
delete[] primaryKeyData;
return ROWLOCATION_INVALID;
}
ret = parsePrimaryKey(blockData + rowLocation.rowOffset, primaryKeyData);
if (ret != SUCCESS) {
delete[] primaryKeyData;
return ret;
}
if (strncmp(primaryKeyData, primaryKey,
schema.getPrimaryKeySchema().length) != 0) {
delete[] primaryKeyData;
return ERROR_TABLE;
}
memcpy(rowData, blockData + rowLocation.rowOffset, schema.getEntrySize());
delete[] primaryKeyData;
return SUCCESS;
}
uint32_t Table::selectColumnItem(const char *primaryKey, const char *columnName,
uint8_t *columnData) {
uint32_t ret = SUCCESS;
uint8_t *rowData = new uint8_t[schema.getEntrySize()];
RowLocation rowLocation;
ret = selectRow(primaryKey, rowData, rowLocation);
if (ret != SUCCESS) {
delete[] rowData;
return ret;
}
uint64_t columnOffset = 0;
uint64_t columnID = 0;
uint32_t columnLength = 0;
ret = getColumnOffset(columnName, columnID, columnLength, columnOffset);
memcpy(columnData, rowData + columnOffset, columnLength);
delete[] rowData;
return SUCCESS;
}
uint32_t Table::updateColumnItem(const char *primaryKey, const char *columnName,
const uint8_t *columnData) {
uint32_t ret = SUCCESS;
uint8_t *blockData = new uint8_t[BLOCK_SIZE];
RowLocation rowLocation;
ret = locateRow(primaryKey, rowLocation);
if (ret != SUCCESS) {
return ret;
}
#ifndef YCSB_TEST
while (prefetchBlockManager.lockBlock(rowLocation.blockID) != SUCCESS)
;
#endif
Driver::read(blockData, rowLocation.blockID);
// char *undoTuple = new char[schema.getEntrySize()];
// memcpy(undoTuple, blockData+rowLocation.rowOffset, schema.getEntrySize());
// UndoLogEntry *logEntry = new UndoLogEntry(this->tableID, primaryKey,
// schema.getPrimaryKeyLength(),
// undoTuple, schema.getEntrySize(),
// rowLocation.blockID, rowLocation.rowOffset);
// buf->append(logEntry);
uint64_t columnOffset, columnID;
uint32_t columnLength;
ret = getColumnOffset(columnName, columnID, columnLength, columnOffset);
uint64_t columnStart = rowLocation.rowOffset + columnOffset;
memcpy(blockData + columnStart, columnData, columnLength);
if (SUCCESS != Driver::write(blockData, rowLocation.blockID)) {
// memcpy(blockData + rowLocation.rowOffset, undoTuple,
// schema.getEntrySize());
// ret = UPDATE_TUPLE_ERROR;
}
// logEntry->expireUndoLog();
#ifndef YCSB_TEST
prefetchBlockManager.unlockBlock(rowLocation.blockID);
#endif
delete[] blockData;
return SUCCESS;
}
uint32_t Table::updateColumnItem_logging(const char *primaryKey,
const char *columnName,
const uint8_t *columnData) {
// TODO
return SUCCESS;
}
uint32_t Table::getColumnOffset(const char *columnName, uint64_t &columnID,
uint32_t &columnLength, uint64_t &offset) {
uint32_t ret = SUCCESS;
offset = 0;
columnID = 0;
columnLength = 0;
ret = getColumnID(columnName, columnID);
if (ret != SUCCESS) {
return ret;
}
std::map<uint64_t, Schema::SchemaEntry> schemaEntrys;
ret = schema.queryAllEntry(schemaEntrys);
if (ret != SUCCESS) {
return ret;
}
columnLength = schemaEntrys[columnID].length;
for (uint64_t i = 1; i < columnID; i++) {
offset += schemaEntrys[i].length;
}
return SUCCESS;
}
uint64_t Table::getRowCount() { return hashTable.getItemCount(); }
uint64_t Table::getTableID() { return tableID; }
char *Table::getTableName() { return tableName; }
uint32_t Table::setTableID(uint64_t tableID) {
this->tableID = tableID;
return SUCCESS;
}
uint32_t Table::setTableName(const char *tableName) {
memcpy(this->tableName, tableName, sizeof(tableName) + 1);
return SUCCESS;
}
uint32_t Table::getColumnID(const char *columnName, uint64_t &columnID) {
return schema.getColumnID(columnName, columnID);
}
Schema &Table::getSchema() { return schema; }
HashTable &Table::getHashTable() { return hashTable; }
} // namespace LightTable