forked from xuos/xiuos
804 lines
31 KiB
Python
804 lines
31 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Copyright JS Foundation and other contributors, http://js.foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import csv
|
|
import itertools
|
|
import os
|
|
import re
|
|
import warnings
|
|
|
|
from gen_c_source import LICENSE, format_code
|
|
from settings import PROJECT_DIR
|
|
|
|
|
|
UNICODE_DATA_FILE = 'UnicodeData.txt'
|
|
SPECIAL_CASING_FILE = 'SpecialCasing.txt'
|
|
DERIVED_PROPS_FILE = 'DerivedCoreProperties.txt'
|
|
PROP_LIST_FILE = 'PropList.txt'
|
|
CASE_FOLDING_FILE = 'CaseFolding.txt'
|
|
|
|
RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
|
|
RANGES_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges-sup.inc.h')
|
|
CONVERSIONS_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
|
|
CONVERSIONS_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions-sup.inc.h')
|
|
FOLDING_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-folding.inc.h')
|
|
FOLDING_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-folding-sup.inc.h')
|
|
|
|
UNICODE_PLANE_TYPE_BASIC = 0
|
|
UNICODE_PLANE_TYPE_SUPPLEMENTARY = 1
|
|
|
|
# common code generation
|
|
|
|
class UnicodeBasicSource(object):
|
|
# pylint: disable=too-many-instance-attributes
|
|
def __init__(self, filepath, character_type="uint16_t", length_type="uint8_t"):
|
|
self._filepath = filepath
|
|
self._header = [LICENSE, ""]
|
|
self._data = []
|
|
self._table_name_suffix = ""
|
|
self.character_type = character_type
|
|
self.length_type = length_type
|
|
|
|
self._range_table_types = [self.character_type,
|
|
self.length_type,
|
|
self.character_type]
|
|
self._range_table_names = ["interval_starts",
|
|
"interval_lengths",
|
|
"chars"]
|
|
self._range_table_descriptions = ["Character interval starting points for",
|
|
"Character interval lengths for",
|
|
"Non-interval characters for"]
|
|
|
|
self._conversion_range_types = [self.character_type,
|
|
self.length_type]
|
|
self._conversion_range_names = ["ranges",
|
|
"range_lengths"]
|
|
|
|
def complete_header(self, completion):
|
|
self._header.append(completion)
|
|
self._header.append("") # for an extra empty line
|
|
|
|
def add_whitepace_range(self, category, categorizer, units):
|
|
self.add_range(category, categorizer.create_tables(units))
|
|
|
|
def add_range(self, category, tables):
|
|
idx = 0
|
|
for table in tables:
|
|
self.add_table(table,
|
|
"/**\n * %s %s.\n */" % (self._range_table_descriptions[idx], category),
|
|
self._range_table_types[idx],
|
|
category,
|
|
self._range_table_names[idx])
|
|
idx += 1
|
|
|
|
def add_conversion_range(self, category, tables, descriptions):
|
|
self.add_named_conversion_range(category, tables, self._conversion_range_names, descriptions)
|
|
|
|
def add_named_conversion_range(self, category, tables, table_names, descriptions):
|
|
idx = 0
|
|
for table in tables:
|
|
self.add_table(table,
|
|
descriptions[idx],
|
|
self._conversion_range_types[idx],
|
|
category,
|
|
table_names[idx])
|
|
idx += 1
|
|
|
|
def add_table(self, table, description, table_type, category, table_name):
|
|
if table and sum(table) != 0:
|
|
self._data.append(description)
|
|
self._data.append("static const %s lit_unicode_%s%s%s[] JERRY_ATTR_CONST_DATA ="
|
|
% (table_type,
|
|
category.lower(),
|
|
"_" + table_name if table_name else "",
|
|
self._table_name_suffix))
|
|
self._data.append("{")
|
|
self._data.append(format_code(table, 1, 6 if self._table_name_suffix else 4))
|
|
self._data.append("};")
|
|
self._data.append("") # for an extra empty line
|
|
|
|
def generate(self):
|
|
with open(self._filepath, 'w') as generated_source:
|
|
generated_source.write("\n".join(self._header))
|
|
generated_source.write("\n".join(self._data))
|
|
|
|
|
|
class UnicodeSupplementarySource(UnicodeBasicSource):
|
|
def __init__(self, filepath):
|
|
UnicodeBasicSource.__init__(self, filepath, "uint32_t", "uint16_t")
|
|
self._table_name_suffix = "_sup"
|
|
|
|
def add_whitepace_range(self, category, categorizer, units):
|
|
self.add_range(category, categorizer.create_tables(units))
|
|
|
|
class UnicodeBasicCategorizer(object):
|
|
def __init__(self):
|
|
self._length_limit = 0xff
|
|
self.extra_id_continue_units = set([0x200C, 0x200D])
|
|
|
|
#pylint: disable=no-self-use
|
|
def in_range(self, i):
|
|
return i >= 0x80 and i < 0x10000
|
|
|
|
def _group_ranges(self, units):
|
|
"""
|
|
Convert an increasing list of integers into a range list
|
|
:return: List of ranges.
|
|
"""
|
|
for _, group in itertools.groupby(enumerate(units), lambda q: (q[1] - q[0])):
|
|
group = list(group)
|
|
yield group[0][1], group[-1][1]
|
|
|
|
def create_tables(self, units):
|
|
"""
|
|
Split list of ranges into intervals and single char lists.
|
|
:return: A tuple containing the following info:
|
|
- list of interval starting points
|
|
- list of interval lengths
|
|
- list of single chars
|
|
"""
|
|
|
|
interval_sps = []
|
|
interval_lengths = []
|
|
chars = []
|
|
|
|
for element in self._group_ranges(units):
|
|
interval_length = element[1] - element[0]
|
|
if interval_length == 0:
|
|
chars.append(element[0])
|
|
elif interval_length > self._length_limit:
|
|
for i in range(element[0], element[1], self._length_limit + 1):
|
|
length = min(self._length_limit, element[1] - i)
|
|
interval_sps.append(i)
|
|
interval_lengths.append(length)
|
|
else:
|
|
interval_sps.append(element[0])
|
|
interval_lengths.append(interval_length)
|
|
|
|
return interval_sps, interval_lengths, chars
|
|
|
|
def read_units(self, file_path, categories, subcategories=None):
|
|
"""
|
|
Read the Unicode Derived Core Properties file and extract the ranges
|
|
for the given categories.
|
|
|
|
:param file_path: Path to the Unicode "DerivedCoreProperties.txt" file.
|
|
:param categories: A list of category strings to extract from the Unicode file.
|
|
:param subcategories: A list of subcategory strings to restrict categories.
|
|
:return: A dictionary each string from the :param categories: is a key and for each
|
|
key list of code points are stored.
|
|
"""
|
|
# Create a dictionary in the format: { category[0]: [ ], ..., category[N]: [ ] }
|
|
units = {}
|
|
for category in categories:
|
|
units[category] = []
|
|
|
|
# Formats to match:
|
|
# <HEX> ; <category> #
|
|
# <HEX>..<HEX> ; <category> # <subcategory>
|
|
matcher = r"(?P<start>[\dA-F]+)(?:\.\.(?P<end>[\dA-F]+))?\s+; (?P<category>[\w]+) # (?P<subcategory>[\w&]{2})"
|
|
|
|
with open(file_path, "r") as src_file:
|
|
for line in src_file:
|
|
match = re.match(matcher, line)
|
|
|
|
if (match
|
|
and match.group("category") in categories
|
|
and (not subcategories or match.group("subcategory") in subcategories)):
|
|
start = int(match.group("start"), 16)
|
|
# if no "end" found use the "start"
|
|
end = int(match.group("end") or match.group("start"), 16)
|
|
|
|
matching_code_points = [
|
|
code_point for code_point in range(start, end + 1) if self.in_range(code_point)
|
|
]
|
|
|
|
units[match.group("category")].extend(matching_code_points)
|
|
|
|
return units
|
|
|
|
def read_case_mappings(self, unicode_data_file, special_casing_file):
|
|
"""
|
|
Read the corresponding unicode values of lower and upper case letters and store these in tables.
|
|
|
|
:param unicode_data_file: Contains the default case mappings (one-to-one mappings).
|
|
:param special_casing_file: Contains additional informative case mappings that are either not one-to-one
|
|
or which are context-sensitive.
|
|
:return: Upper and lower case mappings.
|
|
"""
|
|
|
|
lower_case_mapping = {}
|
|
upper_case_mapping = {}
|
|
|
|
# Add one-to-one mappings
|
|
with open(unicode_data_file) as unicode_data:
|
|
reader = csv.reader(unicode_data, delimiter=';')
|
|
|
|
for line in reader:
|
|
letter_id = int(line[0], 16)
|
|
|
|
if not self.in_range(letter_id):
|
|
continue
|
|
|
|
capital_letter = line[12]
|
|
small_letter = line[13]
|
|
|
|
if capital_letter:
|
|
upper_case_mapping[letter_id] = parse_unicode_sequence(capital_letter)
|
|
|
|
if small_letter:
|
|
lower_case_mapping[letter_id] = parse_unicode_sequence(small_letter)
|
|
|
|
# Update the conversion tables with the special cases
|
|
with open(special_casing_file) as special_casing:
|
|
reader = csv.reader(special_casing, delimiter=';')
|
|
|
|
for line in reader:
|
|
# Skip comment sections and empty lines
|
|
if not line or line[0].startswith('#'):
|
|
continue
|
|
|
|
# Replace '#' character with empty string
|
|
for idx, fragment in enumerate(line):
|
|
if fragment.find('#') >= 0:
|
|
line[idx] = ''
|
|
|
|
letter_id = int(line[0], 16)
|
|
condition_list = line[4]
|
|
|
|
if not self.in_range(letter_id) or condition_list:
|
|
continue
|
|
|
|
original_letter = parse_unicode_sequence(line[0])
|
|
small_letter = parse_unicode_sequence(line[1])
|
|
capital_letter = parse_unicode_sequence(line[3])
|
|
|
|
if small_letter != original_letter:
|
|
lower_case_mapping[letter_id] = small_letter
|
|
if capital_letter != original_letter:
|
|
upper_case_mapping[letter_id] = capital_letter
|
|
|
|
return lower_case_mapping, upper_case_mapping
|
|
|
|
class UnicodeSupplementaryCategorizer(UnicodeBasicCategorizer):
|
|
def __init__(self):
|
|
UnicodeBasicCategorizer.__init__(self)
|
|
self._length_limit = 0xffff
|
|
self.extra_id_continue_units = set()
|
|
|
|
def in_range(self, i):
|
|
return i >= 0x10000
|
|
|
|
def generate_ranges(script_args, plane_type):
|
|
if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
|
|
c_source = UnicodeSupplementarySource(RANGES_SUP_C_SOURCE)
|
|
categorizer = UnicodeSupplementaryCategorizer()
|
|
else:
|
|
c_source = UnicodeBasicSource(RANGES_C_SOURCE)
|
|
categorizer = UnicodeBasicCategorizer()
|
|
|
|
header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
|
|
" * from %s. Do not edit! */" % (DERIVED_PROPS_FILE),
|
|
""]
|
|
|
|
c_source.complete_header("\n".join(header_completion))
|
|
|
|
derived_props_path = os.path.join(script_args.unicode_dir, DERIVED_PROPS_FILE)
|
|
units = categorizer.read_units(derived_props_path, ["ID_Start", "ID_Continue"])
|
|
|
|
units["ID_Continue"] = sorted(set(units["ID_Continue"]).union(categorizer.extra_id_continue_units)
|
|
- set(units["ID_Start"]))
|
|
|
|
for category, unit in units.items():
|
|
c_source.add_range(category, categorizer.create_tables(unit))
|
|
|
|
prop_list_path = os.path.join(script_args.unicode_dir, PROP_LIST_FILE)
|
|
|
|
white_space_units = categorizer.read_units(prop_list_path, ["White_Space"], ["Zs"])["White_Space"]
|
|
|
|
c_source.add_whitepace_range("White_Space", categorizer, white_space_units)
|
|
|
|
c_source.generate()
|
|
|
|
|
|
# functions for unicode conversions
|
|
|
|
def make_char(hex_val):
|
|
"""
|
|
Create a unicode character from a hex value
|
|
|
|
:param hex_val: Hex value of the character.
|
|
:return: Unicode character corresponding to the value.
|
|
"""
|
|
|
|
try:
|
|
return unichr(hex_val)
|
|
except NameError:
|
|
return chr(hex_val)
|
|
|
|
|
|
def parse_unicode_sequence(raw_data):
|
|
"""
|
|
Parse unicode sequence from raw data.
|
|
|
|
:param raw_data: Contains the unicode sequence which needs to parse.
|
|
:return: The parsed unicode sequence.
|
|
"""
|
|
|
|
result = ''
|
|
|
|
for unicode_char in raw_data.split(' '):
|
|
if unicode_char == '':
|
|
continue
|
|
|
|
# Convert it to unicode code point (from hex value without 0x prefix)
|
|
hex_val = int(unicode_char, 16)
|
|
result += make_char(hex_val)
|
|
|
|
return result
|
|
|
|
def extract_ranges(letter_case, reverse_letter_case=None):
|
|
"""
|
|
Extract ranges from case mappings
|
|
(the second param is optional, if it's not empty, a range will contains bidirectional conversions only).
|
|
|
|
:param letter_id: An integer, representing the unicode code point of the character.
|
|
:param letter_case: case mappings dictionary which contains the conversions.
|
|
:param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
|
|
:return: A table with the start points and their mapped value, and another table with the lengths of the ranges.
|
|
"""
|
|
|
|
in_range = False
|
|
range_position = -1
|
|
ranges = []
|
|
range_lengths = []
|
|
|
|
for letter_id in sorted(letter_case.keys()):
|
|
prev_letter_id = letter_id - 1
|
|
|
|
# One-way conversions
|
|
if reverse_letter_case is None:
|
|
if len(letter_case[letter_id]) > 1:
|
|
in_range = False
|
|
continue
|
|
|
|
if prev_letter_id not in letter_case or len(letter_case[prev_letter_id]) > 1:
|
|
in_range = False
|
|
continue
|
|
|
|
# Two way conversions
|
|
else:
|
|
if not is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
|
|
in_range = False
|
|
continue
|
|
|
|
if not is_bidirectional_conversion(prev_letter_id, letter_case, reverse_letter_case):
|
|
in_range = False
|
|
continue
|
|
|
|
conv_distance = calculate_conversion_distance(letter_case, letter_id)
|
|
prev_conv_distance = calculate_conversion_distance(letter_case, prev_letter_id)
|
|
|
|
if conv_distance != prev_conv_distance:
|
|
in_range = False
|
|
continue
|
|
|
|
if in_range:
|
|
range_lengths[range_position] += 1
|
|
else:
|
|
in_range = True
|
|
range_position += 1
|
|
|
|
# Add the start point of the range and its mapped value
|
|
ranges.extend([prev_letter_id, ord(letter_case[prev_letter_id])])
|
|
range_lengths.append(2)
|
|
|
|
# Remove all ranges from the case mapping table.
|
|
for idx in range(0, len(ranges), 2):
|
|
range_length = range_lengths[idx // 2]
|
|
|
|
for incr in range(range_length):
|
|
del letter_case[ranges[idx] + incr]
|
|
if reverse_letter_case is not None:
|
|
del reverse_letter_case[ranges[idx + 1] + incr]
|
|
|
|
return ranges, range_lengths
|
|
|
|
|
|
def extract_character_pair_ranges(letter_case, reverse_letter_case):
|
|
"""
|
|
Extract two or more character pairs from the case mapping tables.
|
|
|
|
:param letter_case: case mappings dictionary which contains the conversions.
|
|
:param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
|
|
:return: A table with the start points, and another table with the lengths of the ranges.
|
|
"""
|
|
|
|
start_points = []
|
|
lengths = []
|
|
in_range = False
|
|
element_counter = -1
|
|
|
|
for letter_id in sorted(letter_case.keys()):
|
|
# Only extract character pairs
|
|
if not is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
|
|
in_range = False
|
|
continue
|
|
|
|
if ord(letter_case[letter_id]) == letter_id + 1:
|
|
prev_letter_id = letter_id - 2
|
|
|
|
if not is_bidirectional_conversion(prev_letter_id, letter_case, reverse_letter_case):
|
|
in_range = False
|
|
|
|
if in_range:
|
|
lengths[element_counter] += 2
|
|
else:
|
|
element_counter += 1
|
|
start_points.append(letter_id)
|
|
lengths.append(2)
|
|
in_range = True
|
|
|
|
else:
|
|
in_range = False
|
|
|
|
# Remove all found case mapping from the conversion tables after the scanning method
|
|
for idx, letter_id in enumerate(start_points):
|
|
conv_length = lengths[idx]
|
|
|
|
for incr in range(0, conv_length, 2):
|
|
del letter_case[letter_id + incr]
|
|
del reverse_letter_case[letter_id + 1 + incr]
|
|
|
|
return start_points, lengths
|
|
|
|
|
|
def extract_character_pairs(letter_case, reverse_letter_case):
|
|
"""
|
|
Extract character pairs. Check that two unicode value are also a mapping value of each other.
|
|
|
|
:param letter_case: case mappings dictionary which contains the conversions.
|
|
:param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
|
|
:return: A table with character pairs.
|
|
"""
|
|
|
|
character_pairs = []
|
|
|
|
for letter_id in sorted(letter_case.keys()):
|
|
if is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
|
|
mapped_value = letter_case[letter_id]
|
|
character_pairs.extend([letter_id, ord(mapped_value)])
|
|
|
|
# Remove character pairs from case mapping tables
|
|
del letter_case[letter_id]
|
|
del reverse_letter_case[ord(mapped_value)]
|
|
|
|
return character_pairs
|
|
|
|
|
|
def extract_special_ranges(letter_case):
|
|
"""
|
|
Extract special ranges. It contains start points of one-to-two letter case ranges
|
|
where the second character is always the same.
|
|
|
|
:param letter_case: case mappings dictionary which contains the conversions.
|
|
|
|
:return: A table with the start points and their mapped values, and a table with the lengths of the ranges.
|
|
"""
|
|
|
|
special_ranges = []
|
|
special_range_lengths = []
|
|
|
|
range_position = -1
|
|
|
|
for letter_id in sorted(letter_case.keys()):
|
|
mapped_value = letter_case[letter_id]
|
|
|
|
if len(mapped_value) != 2:
|
|
continue
|
|
|
|
prev_letter_id = letter_id - 1
|
|
|
|
if prev_letter_id not in letter_case:
|
|
in_range = False
|
|
continue
|
|
|
|
prev_mapped_value = letter_case[prev_letter_id]
|
|
|
|
if len(prev_mapped_value) != 2:
|
|
continue
|
|
|
|
if prev_mapped_value[1] != mapped_value[1]:
|
|
continue
|
|
|
|
if (ord(prev_mapped_value[0]) - prev_letter_id) != (ord(mapped_value[0]) - letter_id):
|
|
in_range = False
|
|
continue
|
|
|
|
if in_range:
|
|
special_range_lengths[range_position] += 1
|
|
else:
|
|
range_position += 1
|
|
in_range = True
|
|
|
|
special_ranges.extend([prev_letter_id, ord(prev_mapped_value[0]), ord(prev_mapped_value[1])])
|
|
special_range_lengths.append(1)
|
|
|
|
# Remove special ranges from the conversion table
|
|
for idx in range(0, len(special_ranges), 3):
|
|
range_length = special_range_lengths[idx // 3]
|
|
letter_id = special_ranges[idx]
|
|
|
|
for incr in range(range_length):
|
|
del letter_case[special_ranges[idx] + incr]
|
|
|
|
return special_ranges, special_range_lengths
|
|
|
|
|
|
def extract_conversions(letter_case):
|
|
"""
|
|
Extract conversions. It provide the full (or remained) case mappings from the table.
|
|
The counter table contains the information of how much one-to-one, one-to-two or one-to-three mappings
|
|
exists successively in the conversion table.
|
|
|
|
:return: A table with conversions, and a table with counters.
|
|
"""
|
|
|
|
unicodes = [[], [], []]
|
|
unicode_lengths = [0, 0, 0]
|
|
|
|
# 1 to 1 byte
|
|
for letter_id in sorted(letter_case.keys()):
|
|
mapped_value = letter_case[letter_id]
|
|
|
|
if len(mapped_value) != 1:
|
|
continue
|
|
|
|
unicodes[0].extend([letter_id, ord(mapped_value)])
|
|
del letter_case[letter_id]
|
|
|
|
# 1 to 2 bytes
|
|
for letter_id in sorted(letter_case.keys()):
|
|
mapped_value = letter_case[letter_id]
|
|
|
|
if len(mapped_value) != 2:
|
|
continue
|
|
|
|
unicodes[1].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1])])
|
|
del letter_case[letter_id]
|
|
|
|
# 1 to 3 bytes
|
|
for letter_id in sorted(letter_case.keys()):
|
|
mapped_value = letter_case[letter_id]
|
|
|
|
if len(mapped_value) != 3:
|
|
continue
|
|
|
|
unicodes[2].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1]), ord(mapped_value[2])])
|
|
del letter_case[letter_id]
|
|
|
|
unicode_lengths = [int(len(unicodes[0]) / 2), int(len(unicodes[1]) / 3), int(len(unicodes[2]) / 4)]
|
|
|
|
return list(itertools.chain.from_iterable(unicodes)), unicode_lengths
|
|
|
|
|
|
def is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
|
|
"""
|
|
Check that two unicode value are also a mapping value of each other.
|
|
|
|
:param letter_id: An integer, representing the unicode code point of the character.
|
|
:param other_case_mapping: Comparable case mapping table which possible contains
|
|
the return direction of the conversion.
|
|
:return: True, if it's a reverible conversion, false otherwise.
|
|
"""
|
|
|
|
if letter_id not in letter_case:
|
|
return False
|
|
|
|
# Check one-to-one mapping
|
|
mapped_value = letter_case[letter_id]
|
|
if len(mapped_value) > 1:
|
|
return False
|
|
|
|
# Check two way conversions
|
|
mapped_value_id = ord(mapped_value)
|
|
|
|
if mapped_value_id not in reverse_letter_case or len(reverse_letter_case[mapped_value_id]) > 1:
|
|
return False
|
|
|
|
if ord(reverse_letter_case[mapped_value_id]) != letter_id:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def calculate_conversion_distance(letter_case, letter_id):
|
|
"""
|
|
Calculate the distance between the unicode character and its mapped value
|
|
(only needs and works with one-to-one mappings).
|
|
|
|
:param letter_case: case mappings dictionary which contains the conversions.
|
|
:param letter_id: An integer, representing the unicode code point of the character.
|
|
:return: The conversion distance.
|
|
"""
|
|
|
|
if letter_id not in letter_case or len(letter_case[letter_id]) > 1:
|
|
return None
|
|
|
|
return ord(letter_case[letter_id]) - letter_id
|
|
|
|
|
|
def generate_conversions(script_args, plane_type):
|
|
if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
|
|
c_source = UnicodeSupplementarySource(CONVERSIONS_SUP_C_SOURCE)
|
|
categorizer = UnicodeSupplementaryCategorizer()
|
|
else:
|
|
c_source = UnicodeBasicSource(CONVERSIONS_C_SOURCE)
|
|
categorizer = UnicodeBasicCategorizer()
|
|
|
|
header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
|
|
" * from %s and %s files. Do not edit! */" % (UNICODE_DATA_FILE, SPECIAL_CASING_FILE),
|
|
""]
|
|
|
|
c_source.complete_header("\n".join(header_completion))
|
|
|
|
unicode_data_path = os.path.join(script_args.unicode_dir, UNICODE_DATA_FILE)
|
|
special_casing_path = os.path.join(script_args.unicode_dir, SPECIAL_CASING_FILE)
|
|
|
|
# Read the corresponding unicode values of lower and upper case letters and store these in tables
|
|
lower_case, upper_case = categorizer.read_case_mappings(unicode_data_path, special_casing_path)
|
|
|
|
c_source.add_conversion_range("character_case",
|
|
extract_ranges(lower_case, upper_case),
|
|
[("/* Contains start points of character case ranges "
|
|
"(these are bidirectional conversions). */"),
|
|
"/* Interval lengths of start points in `character_case_ranges` table. */"])
|
|
c_source.add_conversion_range("character_pair",
|
|
extract_character_pair_ranges(lower_case, upper_case),
|
|
["/* Contains the start points of bidirectional conversion ranges. */",
|
|
"/* Interval lengths of start points in `character_pair_ranges` table. */"])
|
|
|
|
c_source.add_table(extract_character_pairs(lower_case, upper_case),
|
|
"/* Contains lower/upper case bidirectional conversion pairs. */",
|
|
c_source.character_type,
|
|
"character_pairs",
|
|
"")
|
|
|
|
c_source.add_conversion_range("upper_case_special",
|
|
extract_special_ranges(upper_case),
|
|
[("/* Contains start points of one-to-two uppercase ranges where the "
|
|
"second character\n"
|
|
" * is always the same.\n"
|
|
" */"),
|
|
"/* Interval lengths for start points in `upper_case_special_ranges` table. */"])
|
|
|
|
c_source.add_conversion_range("lower_case",
|
|
extract_ranges(lower_case),
|
|
["/* Contains start points of lowercase ranges. */",
|
|
"/* Interval lengths for start points in `lower_case_ranges` table. */"])
|
|
|
|
c_source.add_named_conversion_range("lower_case",
|
|
extract_conversions(lower_case),
|
|
["conversions", "conversion_counters"],
|
|
[("/* The remaining lowercase conversions. The lowercase variant can "
|
|
"be one-to-three character long. */"),
|
|
("/* Number of one-to-one, one-to-two, and one-to-three lowercase "
|
|
"conversions. */")])
|
|
|
|
c_source.add_named_conversion_range("upper_case",
|
|
extract_conversions(upper_case),
|
|
["conversions", "conversion_counters"],
|
|
[("/* The remaining uppercase conversions. The uppercase variant can "
|
|
"be one-to-three character long. */"),
|
|
("/* Number of one-to-one, one-to-two, and one-to-three uppercase "
|
|
"conversions. */")])
|
|
|
|
if lower_case:
|
|
warnings.warn('Not all elements extracted from the lowercase table!')
|
|
if upper_case:
|
|
warnings.warn('Not all elements extracted from the uppercase table!')
|
|
|
|
c_source.generate()
|
|
|
|
|
|
def generate_folding(script_args, plane_type):
|
|
if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
|
|
c_source = UnicodeSupplementarySource(FOLDING_SUP_C_SOURCE)
|
|
categorizer = UnicodeSupplementaryCategorizer()
|
|
else:
|
|
c_source = UnicodeBasicSource(FOLDING_C_SOURCE)
|
|
categorizer = UnicodeBasicCategorizer()
|
|
|
|
header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
|
|
" * from the %s file. Do not edit! */" % (CASE_FOLDING_FILE),
|
|
""]
|
|
|
|
c_source.complete_header("\n".join(header_completion))
|
|
|
|
unicode_data_path = os.path.join(script_args.unicode_dir, UNICODE_DATA_FILE)
|
|
special_casing_path = os.path.join(script_args.unicode_dir, SPECIAL_CASING_FILE)
|
|
case_folding_path = os.path.join(script_args.unicode_dir, CASE_FOLDING_FILE)
|
|
|
|
# Read the corresponding unicode values of lower and upper case letters and store these in tables
|
|
lower_case, upper_case = categorizer.read_case_mappings(unicode_data_path, special_casing_path)
|
|
|
|
folding = {}
|
|
|
|
with open(case_folding_path, 'r') as case_folding:
|
|
case_folding_re = re.compile(r'(?P<code_point>[^;]*);\s*(?P<type>[^;]*);\s*(?P<folding>[^;]*);')
|
|
for line in case_folding:
|
|
match = case_folding_re.match(line)
|
|
if match and match.group('type') in ('S', 'C'):
|
|
code_point = int(match.group('code_point'), 16)
|
|
|
|
if categorizer.in_range(code_point):
|
|
folding[code_point] = parse_unicode_sequence(match.group('folding'))
|
|
|
|
should_to_upper = []
|
|
should_skip_to_lower = []
|
|
|
|
for code_point in lower_case:
|
|
if code_point not in folding:
|
|
should_skip_to_lower.append(code_point)
|
|
|
|
for code_point, folded in folding.items():
|
|
if lower_case.get(code_point, make_char(code_point)) != folded:
|
|
should_to_upper.append(code_point)
|
|
|
|
if upper_case.get(code_point, '') == folded:
|
|
should_skip_to_lower.append(code_point)
|
|
|
|
c_source.add_range('folding_skip_to_lower', categorizer.create_tables(should_skip_to_lower))
|
|
c_source.add_range('folding_to_upper', categorizer.create_tables(should_to_upper))
|
|
|
|
c_source.generate()
|
|
|
|
|
|
# entry point
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='lit-unicode-{conversions,ranges}-{sup}.inc.h generator',
|
|
epilog='''
|
|
The input data must be retrieved from
|
|
http://www.unicode.org/Public/<VERSION>/ucd/UCD.zip.
|
|
The last known good version is 13.0.0.
|
|
''')
|
|
def check_dir(path):
|
|
if not os.path.isdir(path) or not os.access(path, os.R_OK):
|
|
raise argparse.ArgumentTypeError('The %s directory does not exist or is not readable!' % path)
|
|
return path
|
|
|
|
parser.add_argument('--unicode-dir', metavar='DIR', action='store', required=True,
|
|
type=check_dir, help='specify the unicode data directory')
|
|
|
|
script_args = parser.parse_args()
|
|
|
|
generate_ranges(script_args, UNICODE_PLANE_TYPE_BASIC)
|
|
generate_ranges(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)
|
|
generate_conversions(script_args, UNICODE_PLANE_TYPE_BASIC)
|
|
generate_conversions(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)
|
|
generate_folding(script_args, UNICODE_PLANE_TYPE_BASIC)
|
|
# There are currently no code points in the supplementary planes that require special folding
|
|
# generate_folding(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|