diff --git a/Ubiquitous/Kselector_features b/Ubiquitous/Kselector_features new file mode 100644 index 000000000..60de293f0 --- /dev/null +++ b/Ubiquitous/Kselector_features @@ -0,0 +1,65 @@ +menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh) + +menuconfig FS + bool "require file system" + default n + +if FS + config FS_VFS_FATFS + bool "Using FATFS file system" + default n + + config FS_CH376 + bool "Using CH376 file system" + default n + + config FS_LWEXT4 + bool "Using LWEXT4 file system" + default n +endif + + +menuconfig INDUSTRIAL + bool "Require industrial protocols support" + default n + +if INDUSTRIAL + config INDUSTRIAL_OPCUA + bool "Support OPCUA" + default n + + config INDUSTRIAL_SNAP7 + bool "Support Siemens Snap7" + default n + + config INDUSTRIAL_MODBUS + bool "Support Modbus" + default n + + config INDUSTRIAL_RS232 + bool "Support RS232" + default n + + config INDUSTRIAL_RS485 + bool "Support RS485" + default n +endif + + +menuconfig BUS + bool "require bus" + default y + +if BUS + menu "Required BUS" + config USB + bool "require USB bus" + default n + + config SERIAL + bool "require serial bus" + default n + endmenu +endif + +endmenu diff --git a/Ubiquitous/Kselector_features_meta b/Ubiquitous/Kselector_features_meta new file mode 100644 index 000000000..60de293f0 --- /dev/null +++ b/Ubiquitous/Kselector_features_meta @@ -0,0 +1,65 @@ +menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh) + +menuconfig FS + bool "require file system" + default n + +if FS + config FS_VFS_FATFS + bool "Using FATFS file system" + default n + + config FS_CH376 + bool "Using CH376 file system" + default n + + config FS_LWEXT4 + bool "Using LWEXT4 file system" + default n +endif + + +menuconfig INDUSTRIAL + bool "Require industrial protocols support" + default n + +if INDUSTRIAL + config INDUSTRIAL_OPCUA + bool "Support OPCUA" + default n + + config INDUSTRIAL_SNAP7 + bool "Support Siemens Snap7" + default n + + config INDUSTRIAL_MODBUS + bool "Support Modbus" + default n + + config INDUSTRIAL_RS232 + bool "Support RS232" + default n + + config INDUSTRIAL_RS485 + bool "Support RS485" + default n +endif + + +menuconfig BUS + bool "require bus" + default y + +if BUS + menu "Required BUS" + config USB + bool "require USB bus" + default n + + config SERIAL + bool "require serial bus" + default n + endmenu +endif + +endmenu diff --git a/Ubiquitous/Kselector_params b/Ubiquitous/Kselector_params new file mode 100644 index 000000000..3f834ba3f --- /dev/null +++ b/Ubiquitous/Kselector_params @@ -0,0 +1,26 @@ +menu "Required Parameter" + config RAM_LESS_THAN + int "RAM footprint limit (MB) upper bound" + default 10000 + help + Kernel RAM footprint should be lower than this limit + + config RAM_GREATER_THAN + int "RAM footprint limit (MB) lower bound" + default 0 + help + Kernel RAM footprint should be higher than this limit + + config ROM_LESS_THAN + int "ROM limit (MB) upper bound" + default 10000 + help + Kernel ROM requirement should be lower than this limit + + config ROM_GREATER_THAN + int "ROM limit (MB) lower bound" + default 0 + help + Kernel ROM requirement should be higher than this limit + +endmenu diff --git a/Ubiquitous/Kselector_params_meta b/Ubiquitous/Kselector_params_meta new file mode 100644 index 000000000..3f834ba3f --- /dev/null +++ b/Ubiquitous/Kselector_params_meta @@ -0,0 +1,26 @@ +menu "Required Parameter" + config RAM_LESS_THAN + int "RAM footprint limit (MB) upper bound" + default 10000 + help + Kernel RAM footprint should be lower than this limit + + config RAM_GREATER_THAN + int "RAM footprint limit (MB) lower bound" + default 0 + help + Kernel RAM footprint should be higher than this limit + + config ROM_LESS_THAN + int "ROM limit (MB) upper bound" + default 10000 + help + Kernel ROM requirement should be lower than this limit + + config ROM_GREATER_THAN + int "ROM limit (MB) lower bound" + default 0 + help + Kernel ROM requirement should be higher than this limit + +endmenu diff --git a/Ubiquitous/XiUOS/.gitignore b/Ubiquitous/XiUOS/.gitignore index 412a4824c..acc92949c 100644 --- a/Ubiquitous/XiUOS/.gitignore +++ b/Ubiquitous/XiUOS/.gitignore @@ -6,3 +6,7 @@ build XiUOS.* *.swp .vscode +# for kernel_selector +.selector +.selector.old +requirement.yaml \ No newline at end of file diff --git a/Ubiquitous/XiUOS/Makefile b/Ubiquitous/XiUOS/Makefile index 51b129141..4f5b73485 100755 --- a/Ubiquitous/XiUOS/Makefile +++ b/Ubiquitous/XiUOS/Makefile @@ -9,6 +9,8 @@ support :=kd233 stm32f407-st-discovery maix-go stm32f407zgt6 aiit-riscv64-board SRC_DIR:= export BOARD ?=kd233 +# This is the environment variable for kconfig-mconf +export KCONFIG_CONFIG ?= .config ifeq ($(filter $(BOARD),$(support)),) $(warning "You should choose board like this:make BOARD=kd233") @@ -25,11 +27,15 @@ MAKEFILES =$(KERNEL_ROOT)/.config -include $(KERNEL_ROOT)/.config export BSP_ROOT ?= $(KERNEL_ROOT)/board/$(BOARD) +export UBIQUITOUS_ROOT ?= .. include board/$(BOARD)/config.mk export BSP_BUILD_DIR := board/$(BOARD) export HOSTTOOLS_DIR ?= $(KERNEL_ROOT)/tool/hosttools export CONFIG2H_EXE ?= $(HOSTTOOLS_DIR)/xsconfig.sh +export GEN_KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/generate_kselector.py +export FEATURE2YAML_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.sh +export KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.py export CPPPATHS export SRC_APP_DIR := ../../APP_Framework export SRC_KERNEL_DIR := arch board lib fs kernel resources tool @@ -74,6 +80,9 @@ COMPILE_ALL: show_info: + @echo "CONFIG_COMPILER_APP is :" $(CONFIG_COMPILER_APP) + @echo "CONFIG_COMPILER_KERNEL is :" $(CONFIG_COMPILER_KERNEL) + @echo "KERNELPATHS is :" $(KERNELPATHS) @echo "TARGET is :" $(TARGET) @echo "VPATH is :" $(VPATH) @echo "BSP_ROOT is :" $(BSP_ROOT) @@ -100,6 +109,24 @@ menuconfig: @$(CONFIG2H_EXE) .config @cp $(KERNEL_ROOT)/.config $(BSP_ROOT)/.config +kernel_selector: + $(eval KCONFIG_CONFIG := .selector) + @if [ -f "$(BSP_ROOT)/.selector" ]; then \ + cp $(BSP_ROOT)/.selector $(KERNEL_ROOT)/.selector; \ + else if [ -f "$(BSP_ROOT)/.defselector" ]; then \ + cp $(BSP_ROOT)/.defselector $(KERNEL_ROOT)/.selector ;\ + fi ;fi + @$(GEN_KSELECTOR_EXE) + @cp "$(UBIQUITOUS_ROOT)/Kselector_features_meta" $(UBIQUITOUS_ROOT)/Kselector_features + @cp "$(UBIQUITOUS_ROOT)/Kselector_params_meta" $(UBIQUITOUS_ROOT)/Kselector_params + @kconfig-mconf $(BSP_ROOT)/Kselector + @$(FEATURE2YAML_EXE) .selector + @if [ -f "$(BSP_ROOT)/requirement.yaml" ]; then \ + cp $(BSP_ROOT)/requirement.yaml $(KERNEL_ROOT)/requirement.yaml; \ + fi + @$(KSELECTOR_EXE) + @cp $(KERNEL_ROOT)/.selector $(BSP_ROOT)/.selector + clean: @echo Clean target and build_dir @rm -rf build diff --git a/Ubiquitous/XiUOS/board/aiit-arm32-board/Kselector b/Ubiquitous/XiUOS/board/aiit-arm32-board/Kselector new file mode 100644 index 000000000..f60f898f7 --- /dev/null +++ b/Ubiquitous/XiUOS/board/aiit-arm32-board/Kselector @@ -0,0 +1,40 @@ + +mainmenu "Ubiquitous Kernel Selector" + +config BSP_DIR + string + option env="BSP_ROOT" + default "." + +config KERNEL_DIR + string + option env="KERNEL_ROOT" + default "../.." + +config UBIQUITOUS_DIR + string + option env="UBIQUITOUS_ROOT" + default "../../.." + +source "$UBIQUITOUS_DIR/Kselector_features" + +source "$UBIQUITOUS_DIR/Kselector_params" + +config MANUALLY_SELECT + bool "Manually select a kernel" + default n +if MANUALLY_SELECT +menu "Required Kernel" +choice + prompt "Select OS Kernel" + default SELECT_XIUOS + + config SELECT_RT_THREAD + bool "select RT_Thread" + config SELECT_NUTTX + bool "select Nuttx" + config SELECT_XIUOS + bool "select XiUOS" +endchoice +endmenu +endif diff --git a/Ubiquitous/XiUOS/tool/hosttools/generate_kselector.py b/Ubiquitous/XiUOS/tool/hosttools/generate_kselector.py new file mode 100755 index 000000000..8f74a12e8 --- /dev/null +++ b/Ubiquitous/XiUOS/tool/hosttools/generate_kselector.py @@ -0,0 +1,73 @@ +#! /usr/bin/python3 + +import os + +ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT') +bsp_dir = os.environ.get('BSP_ROOT') +kernel_names = [] + +def get_kernel_names(): + for d in os.scandir(ubiquitous_dir): + if d.is_dir() and d.name!= "feature_yaml": + kernel_names.append(d.name) + +def generate_features(): + with open(f"{ubiquitous_dir}/Kselector_features","w") as f: + # f.write("") + pass + +def generate_params(): + with open(f"{ubiquitous_dir}/Kselector_params","w") as f: + # f.write("") + pass + +def generate(): + get_kernel_names() + generate_features() + generate_params() + + template = r""" +mainmenu "Ubiquitous Kernel Selector" + +config BSP_DIR + string + option env="BSP_ROOT" + default "." + +config KERNEL_DIR + string + option env="KERNEL_ROOT" + default "../.." + +config UBIQUITOUS_DIR + string + option env="UBIQUITOUS_ROOT" + default "../../.." + +source "$UBIQUITOUS_DIR/Kselector_features" + +source "$UBIQUITOUS_DIR/Kselector_params" + +config MANUALLY_SELECT + bool "Manually select a kernel" + default n +if MANUALLY_SELECT +menu "Required Kernel" +choice + prompt "Select OS Kernel" + default SELECT_XIUOS + + """ + + for kernel_name in kernel_names: + template += f"\tconfig SELECT_{kernel_name.upper()}\n" + template += f'\t\tbool "select {kernel_name}"\n' + + # the last `\n` is very important, otherwise, it cannot be recognized as valid Kconfig file + template += "endchoice\nendmenu\nendif\n" + + with open(f"{bsp_dir}/Kselector", "w") as f: + f.write(template) + +if __name__ == '__main__': + generate() \ No newline at end of file diff --git a/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.py b/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.py new file mode 100755 index 000000000..428ee9953 --- /dev/null +++ b/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.py @@ -0,0 +1,159 @@ +#! /usr/bin/python3 +# this script builds a decision tree to select suitable kernels from developers' preferences +from __future__ import annotations +from typing import TypeVar, Generic, Tuple, Optional +import os +import yaml +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT') +kernel_dir = os.environ.get('KERNEL_ROOT') + +YAMLS_PATH = f"{ubiquitous_dir}/feature_yaml/" +REQUIREMENT_PATH = f"{kernel_dir}/requirement.yaml" + +COMPONENT_KEY = "Feature" +PARAMETER_KEY = "Parameter" +MANUAL_KEY = "Kernel" + +# tree node for kernel features +V = TypeVar("V") +class TreeNode(Generic[V]): + def __init__(self, depth: int, name: str, value: V): + self.depth = depth + self.name = name + self.value = value + self.children = dict() + + def __repr__(self): + res = f"L{self.depth}: name {self.name} - value {self.value} - {len(self.children)} children\n" + for c in self.children: + # recursively build representation + res += self.children[c].__repr__() + return res + + def node_number(self): + if len(self.children) == 0: + return 1 + number = 1 + for c in self.children: + number += self.children[c].node_number() + return number + + +# this class should be constructed from the yaml file +class Kernel: + def __init__(self, name, feature_yaml: dict): + self.name = name + self.features_root: Optional[TreeNode[bool]] = None + self.parameters_root: Optional[TreeNode[int]] = None + self.extract_components(feature_yaml) + self.extract_parameters(feature_yaml) + + def __repr__(self): + return f"{self.name}\nFeature Tree:\n{self.features_root}Parameter Tree:\n{self.parameters_root}\n" + + def extract_components(self, feature_yaml: dict): + self.features_root = TreeNode(0, COMPONENT_KEY, True) + for k,v in feature_yaml[COMPONENT_KEY].items(): + node = TreeNode(1, k, v) + if k in feature_yaml: + self.build_sub_tree(2, node, feature_yaml) + self.features_root.children[k] = node + + def build_sub_tree(self, depth, node: TreeNode[bool], feature_yaml: dict): + if feature_yaml[node.name] is None: + return + for k,v in feature_yaml[node.name].items(): + child = TreeNode(depth, k, v) + if child.name in feature_yaml: + self.build_sub_tree(depth+1, child, feature_yaml) + node.children[k] = child + + def extract_parameters(self, feature_yaml: dict): + self.parameters_root = TreeNode(0, PARAMETER_KEY, 0) + for k,v in feature_yaml[PARAMETER_KEY].items(): + child = TreeNode(1, k, v) + self.parameters_root.children[k] = child + + def check(self, requirements_root, root) -> Tuple[bool, int]: + distance = 0 + q = list() + q.append((requirements_root, root)) + while len(q) > 0: + (r_node, s_node) = q.pop() + for c in r_node.children: + if c in s_node.children and self.compare(c, s_node.children[c].value, r_node.children[c].value): + q.append((r_node.children[c], s_node.children[c])) + distance += (r_node.children[c].value - s_node.children[c].value)**2 + else: + return False, distance + return True, distance + + def compare(self, name, v1, v2): + if name.endswith("GREATER_THAN"): + return v1 >= v2 + elif name.endswith("LESS_THAN"): + return v1 <= v2 + else: + return v1 == v2 + +# load the requirements from the yaml file +def load_required_features()->dict: + with open(REQUIREMENT_PATH, 'r') as f: + content = f.read() + feature_yaml = yaml.load(content, Loader=Loader) + print(feature_yaml) + return feature_yaml + +# load kernel yamls from ubiquitous directory and build Kernel class +def load_kernel_features()->list[Kernel]: + kernels = {} + for file_name in os.listdir(YAMLS_PATH): + kernel_name = file_name.rstrip(".yaml") + with open(os.path.join(YAMLS_PATH, file_name), 'r') as f: + content = f.read() + feature_yaml = yaml.load(content, Loader=Loader) + kernels[kernel_name] = Kernel(kernel_name, feature_yaml) + return kernels + +# recommend a suitable kernel according to the requirement +def select_kernel(requirement: dict, kernels: dict[Kernel]) -> str: + selected = None + selected_node_number = 0 + selected_distance = 0 + requirement = Kernel("requirement", requirement) + for name, kernel in kernels.items(): + # here should be a tree matching algorithm + # requirement trees (both features and parameters) + # should be subtrees of the recommended kernel + pass_features, _ = kernel.check(requirement.features_root, kernel.features_root) + pass_parameters, distance = kernel.check(requirement.parameters_root, kernel.parameters_root) + print(name, pass_features, pass_parameters, distance) + if pass_features and pass_parameters: + if selected is None: + selected = kernel + selected_node_number = kernel.features_root.node_number() + selected_node_number = distance + else: + node_number = kernel.features_root.node_number() + if selected_node_number > node_number or (selected_node_number == node_number and selected_distance > distance): + selected = kernel + selected_node_number = kernel.features_root.node_number() + selected_node_number = distance + if selected is None: + return "Cannot find suitable kernel" + else: + return selected.name + +if __name__ == "__main__": + requirement = load_required_features() + if MANUAL_KEY in requirement: + selected = list(requirement[MANUAL_KEY].keys())[0][7:] + else: + kernels = load_kernel_features() + selected = select_kernel(requirement, kernels) + print(f"Selected kernel is: {selected}") \ No newline at end of file diff --git a/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.sh b/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.sh new file mode 100755 index 000000000..eb8bd3df4 --- /dev/null +++ b/Ubiquitous/XiUOS/tool/hosttools/kernel_selector.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# use this script to read the `.feature file` generated from `kconfig-mconf `, which further parse Kfeature to build the menu +# this script will genearte a more structured yaml file, +# the json file would further be sent to a decision tree for kernel selection. + +VERSION=0.1.0 + +function generate_requirement_file() +{ + local SELECTOR_NAME=${1} + + # destination file using file descriptor 8 + exec 8>${2} + + echo -ne "version: ${VERSION}\n" >&8 + + EMPTY_LINE='true' + + while read LN + do + LINE=`echo $LN | sed 's/[ \t\r\n]*$//g'` + + if [ -z "$LINE" ]; then + continue + fi + + if [ '#' = ${LINE:0:1} ]; then + if [ ${#LINE} -eq 1 ]; then + # empty line + if $EMPTY_LINE; then + continue + fi + echo >&8 + EMPTY_LINE='true' + continue + fi + + if echo -ne "$LINE" | egrep '^# Required' >/dev/null 2>/dev/null; then + # the key in yaml + echo -ne "${LINE:11}:\n" >&8 + else + LINE=${LINE:1} + echo -ne "# ${LINE}\n" >&8 + fi + + EMPTY_LINE='false' + else + EMPTY_LINE='false' + + OLD_IFS="$IFS" + IFS='=' + REQUIREMENTS=($LINE) + IFS="$OLD_IFS" + + if [ ${#REQUIREMENTS[@]} -ge 2 ]; then + if echo -ne "$REQUIREMENTS[0]" | egrep '^CONFIG_' >/dev/null 2>/dev/null; then + REQUIREMENTS[0]="${REQUIREMENTS[0]:7}" + fi + + if [ "${REQUIREMENTS[1]}" = 'y' ]; then + echo -ne " ${REQUIREMENTS[0]}: true\n" >&8 + else + echo -ne " ${REQUIREMENTS[0]}: ${LINE#*=}\n" >&8 + fi + fi + fi + + done < $SELECTOR_NAME + + exec 8<&- +} + +generate_requirement_file $1 $BSP_ROOT/requirement.yaml diff --git a/Ubiquitous/feature_yaml/Nuttx.yaml b/Ubiquitous/feature_yaml/Nuttx.yaml new file mode 100755 index 000000000..01879de18 --- /dev/null +++ b/Ubiquitous/feature_yaml/Nuttx.yaml @@ -0,0 +1,21 @@ +version: 0.1.0 +# Automatically generated file; DO NOT EDIT. +# Ubiquitous Kernel Selector + +Feature: + +# CONFIG_FS is not set + BUS: true + +BUS: + + USB: true +# CONFIG_SERIAL is not set + +Parameter: + + RAM_LESS_THAN: 20 + RAM_GREATER_THAN: 10000 + ROM_LESS_THAN: 10 + ROM_GREATER_THAN: 10000 + diff --git a/Ubiquitous/feature_yaml/RT_Thread.yaml b/Ubiquitous/feature_yaml/RT_Thread.yaml new file mode 100755 index 000000000..a1d1ce726 --- /dev/null +++ b/Ubiquitous/feature_yaml/RT_Thread.yaml @@ -0,0 +1,21 @@ +version: 0.1.0 +# Automatically generated file; DO NOT EDIT. +# Ubiquitous Kernel Selector + +Feature: + +# CONFIG_FS is not set + BUS: true + +BUS: + + USB: true +# CONFIG_SERIAL is not set + +Parameter: + + RAM_LESS_THAN: 5 + RAM_GREATER_THAN: 10000 + ROM_LESS_THAN: 5 + ROM_GREATER_THAN: 10000 + diff --git a/Ubiquitous/feature_yaml/XiUOS.yaml b/Ubiquitous/feature_yaml/XiUOS.yaml new file mode 100755 index 000000000..2e9c4b8c4 --- /dev/null +++ b/Ubiquitous/feature_yaml/XiUOS.yaml @@ -0,0 +1,21 @@ +version: 0.1.0 +# Automatically generated file; DO NOT EDIT. +# Ubiquitous Kernel Selector + +Feature: + +# CONFIG_FS is not set + BUS: true + +BUS: + + USB: true +# CONFIG_SERIAL is not set + +Parameter: + + RAM_LESS_THAN: 3 + RAM_GREATER_THAN: 10000 + ROM_LESS_THAN: 10 + ROM_GREATER_THAN: 10000 +