developer
/
xiuos
forked from xuos/xiuos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build a kernel selector tool in Kconfig/bash/python. Implement a naive recommendation algorithm for it. Automatically generate preferences in YAML. The features described in YAML for each kernel is not stable and is only for demonstration currently. Integrate the tool into the Makefile of XiUOS. Run make kernel_selector in XiUOS directory to verify it

This commit is contained in:
xuedongliang 2022-02-24 16:18:37 +08:00
parent f1e8216944 60cd74e296
commit 4f5d2fc070
13 changed files with 621 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
Ubiquitous/Kselector_features Normal file
View File

@ -0,0 +1,65 @@
menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh)
menuconfig FS
bool "require file system"
default n
if FS
config FS_VFS_FATFS
bool "Using FATFS file system"
default n
config FS_CH376
bool "Using CH376 file system"
default n
config FS_LWEXT4
bool "Using LWEXT4 file system"
default n
endif
menuconfig INDUSTRIAL
bool "Require industrial protocols support"
default n
if INDUSTRIAL
config INDUSTRIAL_OPCUA
bool "Support OPCUA"
default n
config INDUSTRIAL_SNAP7
bool "Support Siemens Snap7"
default n
config INDUSTRIAL_MODBUS
bool "Support Modbus"
default n
config INDUSTRIAL_RS232
bool "Support RS232"
default n
config INDUSTRIAL_RS485
bool "Support RS485"
default n
endif
menuconfig BUS
bool "require bus"
default y
if BUS
menu "Required BUS"
config USB
bool "require USB bus"
default n
config SERIAL
bool "require serial bus"
default n
endmenu
endif
endmenu

65
Ubiquitous/Kselector_features_meta Normal file
View File

@ -0,0 +1,65 @@
menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh)
menuconfig FS
bool "require file system"
default n
if FS
config FS_VFS_FATFS
bool "Using FATFS file system"
default n
config FS_CH376
bool "Using CH376 file system"
default n
config FS_LWEXT4
bool "Using LWEXT4 file system"
default n
endif
menuconfig INDUSTRIAL
bool "Require industrial protocols support"
default n
if INDUSTRIAL
config INDUSTRIAL_OPCUA
bool "Support OPCUA"
default n
config INDUSTRIAL_SNAP7
bool "Support Siemens Snap7"
default n
config INDUSTRIAL_MODBUS
bool "Support Modbus"
default n
config INDUSTRIAL_RS232
bool "Support RS232"
default n
config INDUSTRIAL_RS485
bool "Support RS485"
default n
endif
menuconfig BUS
bool "require bus"
default y
if BUS
menu "Required BUS"
config USB
bool "require USB bus"
default n
config SERIAL
bool "require serial bus"
default n
endmenu
endif
endmenu

26
Ubiquitous/Kselector_params Normal file
View File

@ -0,0 +1,26 @@
menu "Required Parameter"
config RAM_LESS_THAN
int "RAM footprint limit (MB) upper bound"
default 10000
help
Kernel RAM footprint should be lower than this limit
config RAM_GREATER_THAN
int "RAM footprint limit (MB) lower bound"
default 0
help
Kernel RAM footprint should be higher than this limit
config ROM_LESS_THAN
int "ROM limit (MB) upper bound"
default 10000
help
Kernel ROM requirement should be lower than this limit
config ROM_GREATER_THAN
int "ROM limit (MB) lower bound"
default 0
help
Kernel ROM requirement should be higher than this limit
endmenu

26
Ubiquitous/Kselector_params_meta Normal file
View File

@ -0,0 +1,26 @@
menu "Required Parameter"
config RAM_LESS_THAN
int "RAM footprint limit (MB) upper bound"
default 10000
help
Kernel RAM footprint should be lower than this limit
config RAM_GREATER_THAN
int "RAM footprint limit (MB) lower bound"
default 0
help
Kernel RAM footprint should be higher than this limit
config ROM_LESS_THAN
int "ROM limit (MB) upper bound"
default 10000
help
Kernel ROM requirement should be lower than this limit
config ROM_GREATER_THAN
int "ROM limit (MB) lower bound"
default 0
help
Kernel ROM requirement should be higher than this limit
endmenu

4
Ubiquitous/XiUOS/.gitignore vendored
View File

@ -6,3 +6,7 @@ build
XiUOS.*
*.swp
.vscode
# for kernel_selector
.selector
.selector.old
requirement.yaml

27
Ubiquitous/XiUOS/Makefile
View File

@ -9,6 +9,8 @@ support :=kd233 stm32f407-st-discovery maix-go stm32f407zgt6 aiit-riscv64-board
SRC_DIR:=
export BOARD ?=kd233
# This is the environment variable for kconfig-mconf
export KCONFIG_CONFIG ?= .config
ifeq ($(filter $(BOARD),$(support)),)
$(warning "You should choose board like this:make BOARD=kd233")
@ -25,11 +27,15 @@ MAKEFILES =$(KERNEL_ROOT)/.config
-include $(KERNEL_ROOT)/.config
export BSP_ROOT ?= $(KERNEL_ROOT)/board/$(BOARD)
export UBIQUITOUS_ROOT ?= ..
include board/$(BOARD)/config.mk
export BSP_BUILD_DIR := board/$(BOARD)
export HOSTTOOLS_DIR ?= $(KERNEL_ROOT)/tool/hosttools
export CONFIG2H_EXE ?= $(HOSTTOOLS_DIR)/xsconfig.sh
export GEN_KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/generate_kselector.py
export FEATURE2YAML_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.sh
export KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.py
export CPPPATHS
export SRC_APP_DIR := ../../APP_Framework
export SRC_KERNEL_DIR := arch board lib fs kernel resources tool
@ -74,6 +80,9 @@ COMPILE_ALL:
show_info:
@echo "CONFIG_COMPILER_APP is :" $(CONFIG_COMPILER_APP)
@echo "CONFIG_COMPILER_KERNEL is :" $(CONFIG_COMPILER_KERNEL)
@echo "KERNELPATHS is :" $(KERNELPATHS)
@echo "TARGET is :" $(TARGET)
@echo "VPATH is :" $(VPATH)
@echo "BSP_ROOT is :" $(BSP_ROOT)
@ -100,6 +109,24 @@ menuconfig:
@$(CONFIG2H_EXE) .config
@cp $(KERNEL_ROOT)/.config $(BSP_ROOT)/.config
kernel_selector:
$(eval KCONFIG_CONFIG := .selector)
@if [ -f "$(BSP_ROOT)/.selector" ]; then \
cp $(BSP_ROOT)/.selector $(KERNEL_ROOT)/.selector; \
else if [ -f "$(BSP_ROOT)/.defselector" ]; then \
cp $(BSP_ROOT)/.defselector $(KERNEL_ROOT)/.selector ;\
fi ;fi
@$(GEN_KSELECTOR_EXE)
@cp "$(UBIQUITOUS_ROOT)/Kselector_features_meta" $(UBIQUITOUS_ROOT)/Kselector_features
@cp "$(UBIQUITOUS_ROOT)/Kselector_params_meta" $(UBIQUITOUS_ROOT)/Kselector_params
@kconfig-mconf $(BSP_ROOT)/Kselector
@$(FEATURE2YAML_EXE) .selector
@if [ -f "$(BSP_ROOT)/requirement.yaml" ]; then \
cp $(BSP_ROOT)/requirement.yaml $(KERNEL_ROOT)/requirement.yaml; \
fi
@$(KSELECTOR_EXE)
@cp $(KERNEL_ROOT)/.selector $(BSP_ROOT)/.selector
clean:
@echo Clean target and build_dir
@rm -rf build

40
Ubiquitous/XiUOS/board/aiit-arm32-board/Kselector Normal file
View File

@ -0,0 +1,40 @@
mainmenu "Ubiquitous Kernel Selector"
config BSP_DIR
string
option env="BSP_ROOT"
default "."
config KERNEL_DIR
string
option env="KERNEL_ROOT"
default "../.."
config UBIQUITOUS_DIR
string
option env="UBIQUITOUS_ROOT"
default "../../.."
source "$UBIQUITOUS_DIR/Kselector_features"
source "$UBIQUITOUS_DIR/Kselector_params"
config MANUALLY_SELECT
bool "Manually select a kernel"
default n
if MANUALLY_SELECT
menu "Required Kernel"
choice
prompt "Select OS Kernel"
default SELECT_XIUOS
config SELECT_RT_THREAD
bool "select RT_Thread"
config SELECT_NUTTX
bool "select Nuttx"
config SELECT_XIUOS
bool "select XiUOS"
endchoice
endmenu
endif

73
Ubiquitous/XiUOS/tool/hosttools/generate_kselector.py Executable file
View File

@ -0,0 +1,73 @@
#! /usr/bin/python3
import os
ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT')
bsp_dir = os.environ.get('BSP_ROOT')
kernel_names = []
def get_kernel_names():
for d in os.scandir(ubiquitous_dir):
if d.is_dir() and d.name!= "feature_yaml":
kernel_names.append(d.name)
def generate_features():
with open(f"{ubiquitous_dir}/Kselector_features","w") as f:
# f.write("")
pass
def generate_params():
with open(f"{ubiquitous_dir}/Kselector_params","w") as f:
# f.write("")
pass
def generate():
get_kernel_names()
generate_features()
generate_params()
template = r"""
mainmenu "Ubiquitous Kernel Selector"
config BSP_DIR
string
option env="BSP_ROOT"
default "."
config KERNEL_DIR
string
option env="KERNEL_ROOT"
default "../.."
config UBIQUITOUS_DIR
string
option env="UBIQUITOUS_ROOT"
default "../../.."
source "$UBIQUITOUS_DIR/Kselector_features"
source "$UBIQUITOUS_DIR/Kselector_params"
config MANUALLY_SELECT
bool "Manually select a kernel"
default n
if MANUALLY_SELECT
menu "Required Kernel"
choice
prompt "Select OS Kernel"
default SELECT_XIUOS
"""
for kernel_name in kernel_names:
template += f"\tconfig SELECT_{kernel_name.upper()}\n"
template += f'\t\tbool "select {kernel_name}"\n'
# the last `\n` is very important, otherwise, it cannot be recognized as valid Kconfig file
template += "endchoice\nendmenu\nendif\n"
with open(f"{bsp_dir}/Kselector", "w") as f:
f.write(template)
if __name__ == '__main__':
generate()

159
Ubiquitous/XiUOS/tool/hosttools/kernel_selector.py Executable file
View File

@ -0,0 +1,159 @@
#! /usr/bin/python3
# this script builds a decision tree to select suitable kernels from developers' preferences
from __future__ import annotations
from typing import TypeVar, Generic, Tuple, Optional
import os
import yaml
try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT')
kernel_dir = os.environ.get('KERNEL_ROOT')
YAMLS_PATH = f"{ubiquitous_dir}/feature_yaml/"
REQUIREMENT_PATH = f"{kernel_dir}/requirement.yaml"
COMPONENT_KEY = "Feature"
PARAMETER_KEY = "Parameter"
MANUAL_KEY = "Kernel"
# tree node for kernel features
V = TypeVar("V")
class TreeNode(Generic[V]):
def __init__(self, depth: int, name: str, value: V):
self.depth = depth
self.name = name
self.value = value
self.children = dict()
def __repr__(self):
res = f"L{self.depth}: name {self.name} - value {self.value} - {len(self.children)} children\n"
for c in self.children:
# recursively build representation
res += self.children[c].__repr__()
return res
def node_number(self):
if len(self.children) == 0:
return 1
number = 1
for c in self.children:
number += self.children[c].node_number()
return number
# this class should be constructed from the yaml file
class Kernel:
def __init__(self, name, feature_yaml: dict):
self.name = name
self.features_root: Optional[TreeNode[bool]] = None
self.parameters_root: Optional[TreeNode[int]] = None
self.extract_components(feature_yaml)
self.extract_parameters(feature_yaml)
def __repr__(self):
return f"{self.name}\nFeature Tree:\n{self.features_root}Parameter Tree:\n{self.parameters_root}\n"
def extract_components(self, feature_yaml: dict):
self.features_root = TreeNode(0, COMPONENT_KEY, True)
for k,v in feature_yaml[COMPONENT_KEY].items():
node = TreeNode(1, k, v)
if k in feature_yaml:
self.build_sub_tree(2, node, feature_yaml)
self.features_root.children[k] = node
def build_sub_tree(self, depth, node: TreeNode[bool], feature_yaml: dict):
if feature_yaml[node.name] is None:
return
for k,v in feature_yaml[node.name].items():
child = TreeNode(depth, k, v)
if child.name in feature_yaml:
self.build_sub_tree(depth+1, child, feature_yaml)
node.children[k] = child
def extract_parameters(self, feature_yaml: dict):
self.parameters_root = TreeNode(0, PARAMETER_KEY, 0)
for k,v in feature_yaml[PARAMETER_KEY].items():
child = TreeNode(1, k, v)
self.parameters_root.children[k] = child
def check(self, requirements_root, root) -> Tuple[bool, int]:
distance = 0
q = list()
q.append((requirements_root, root))
while len(q) > 0:
(r_node, s_node) = q.pop()
for c in r_node.children:
if c in s_node.children and self.compare(c, s_node.children[c].value, r_node.children[c].value):
q.append((r_node.children[c], s_node.children[c]))
distance += (r_node.children[c].value - s_node.children[c].value)**2
else:
return False, distance
return True, distance
def compare(self, name, v1, v2):
if name.endswith("GREATER_THAN"):
return v1 >= v2
elif name.endswith("LESS_THAN"):
return v1 <= v2
else:
return v1 == v2
# load the requirements from the yaml file
def load_required_features()->dict:
with open(REQUIREMENT_PATH, 'r') as f:
content = f.read()
feature_yaml = yaml.load(content, Loader=Loader)
print(feature_yaml)
return feature_yaml
# load kernel yamls from ubiquitous directory and build Kernel class
def load_kernel_features()->list[Kernel]:
kernels = {}
for file_name in os.listdir(YAMLS_PATH):
kernel_name = file_name.rstrip(".yaml")
with open(os.path.join(YAMLS_PATH, file_name), 'r') as f:
content = f.read()
feature_yaml = yaml.load(content, Loader=Loader)
kernels[kernel_name] = Kernel(kernel_name, feature_yaml)
return kernels
# recommend a suitable kernel according to the requirement
def select_kernel(requirement: dict, kernels: dict[Kernel]) -> str:
selected = None
selected_node_number = 0
selected_distance = 0
requirement = Kernel("requirement", requirement)
for name, kernel in kernels.items():
# here should be a tree matching algorithm
# requirement trees (both features and parameters)
# should be subtrees of the recommended kernel
pass_features, _ = kernel.check(requirement.features_root, kernel.features_root)
pass_parameters, distance = kernel.check(requirement.parameters_root, kernel.parameters_root)
print(name, pass_features, pass_parameters, distance)
if pass_features and pass_parameters:
if selected is None:
selected = kernel
selected_node_number = kernel.features_root.node_number()
selected_node_number = distance
else:
node_number = kernel.features_root.node_number()
if selected_node_number > node_number or (selected_node_number == node_number and selected_distance > distance):
selected = kernel
selected_node_number = kernel.features_root.node_number()
selected_node_number = distance
if selected is None:
return "Cannot find suitable kernel"
else:
return selected.name
if __name__ == "__main__":
requirement = load_required_features()
if MANUAL_KEY in requirement:
selected = list(requirement[MANUAL_KEY].keys())[0][7:]
else:
kernels = load_kernel_features()
selected = select_kernel(requirement, kernels)
print(f"Selected kernel is: {selected}")

73
Ubiquitous/XiUOS/tool/hosttools/kernel_selector.sh Executable file
View File

@ -0,0 +1,73 @@
#!/bin/bash
# use this script to read the `.feature file` generated from `kconfig-mconf `, which further parse Kfeature to build the menu
# this script will genearte a more structured yaml file,
# the json file would further be sent to a decision tree for kernel selection.
VERSION=0.1.0
function generate_requirement_file()
{
local SELECTOR_NAME=${1}
# destination file using file descriptor 8
exec 8>${2}
echo -ne "version: ${VERSION}\n" >&8
EMPTY_LINE='true'
while read LN
do
LINE=`echo $LN | sed 's/[ \t\r\n]*$//g'`
if [ -z "$LINE" ]; then
continue
fi
if [ '#' = ${LINE:0:1} ]; then
if [ ${#LINE} -eq 1 ]; then
# empty line
if $EMPTY_LINE; then
continue
fi
echo >&8
EMPTY_LINE='true'
continue
fi
if echo -ne "$LINE" | egrep '^# Required' >/dev/null 2>/dev/null; then
# the key in yaml
echo -ne "${LINE:11}:\n" >&8
else
LINE=${LINE:1}
echo -ne "# ${LINE}\n" >&8
fi
EMPTY_LINE='false'
else
EMPTY_LINE='false'
OLD_IFS="$IFS"
IFS='='
REQUIREMENTS=($LINE)
IFS="$OLD_IFS"
if [ ${#REQUIREMENTS[@]} -ge 2 ]; then
if echo -ne "$REQUIREMENTS[0]" | egrep '^CONFIG_' >/dev/null 2>/dev/null; then
REQUIREMENTS[0]="${REQUIREMENTS[0]:7}"
fi
if [ "${REQUIREMENTS[1]}" = 'y' ]; then
echo -ne " ${REQUIREMENTS[0]}: true\n" >&8
else
echo -ne " ${REQUIREMENTS[0]}: ${LINE#*=}\n" >&8
fi
fi
fi
done < $SELECTOR_NAME
exec 8<&-
}
generate_requirement_file $1 $BSP_ROOT/requirement.yaml

21
Ubiquitous/feature_yaml/Nuttx.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 20
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 10
ROM_GREATER_THAN: 10000

21
Ubiquitous/feature_yaml/RT_Thread.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 5
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 5
ROM_GREATER_THAN: 10000

21
Ubiquitous/feature_yaml/XiUOS.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 3
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 10
ROM_GREATER_THAN: 10000