Fix locale scripts (#2648)

Co-authored-by: mhamster <81412348+mhamsterr@users.noreply.github.com>
Co-authored-by: Morb0 <14136326+Morb0@users.noreply.github.com>

Tools/ss14_ru/__init__.py

@@ -1 +1 @@
-from fluentformatter import FluentFile, FluentFormatter
+from fluentformatter import FluentFile, FluentFormatter

Tools/ss14_ru/clean_duplicates.py

@@ -0,0 +1,120 @@
+import os
+import re
+import chardet
+from datetime import datetime
+
+def find_top_level_dir(start_dir):
+    marker_file = 'SpaceStation14.sln'
+    current_dir = start_dir
+    while True:
+        if marker_file in os.listdir(current_dir):
+            return current_dir
+        parent_dir = os.path.dirname(current_dir)
+        if parent_dir == current_dir:
+            print(f"Не удалось найти {marker_file} начиная с {start_dir}")
+            exit(-1)
+        current_dir = parent_dir
+        
+def find_ftl_files(root_dir):
+    ftl_files = []
+    for root, dirs, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith('.ftl'):
+                ftl_files.append(os.path.join(root, file))
+    return ftl_files
+
+def detect_encoding(file_path):
+    with open(file_path, 'rb') as file:
+        raw_data = file.read()
+    return chardet.detect(raw_data)['encoding']
+
+def parse_ent_blocks(file_path):
+    try:
+        encoding = detect_encoding(file_path)
+        with open(file_path, 'r', encoding=encoding) as file:
+            content = file.read()
+    except UnicodeDecodeError:
+        print(f"Ошибка при чтении файла {file_path}. Попытка чтения в UTF-8.")
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                content = file.read()
+        except UnicodeDecodeError:
+            print(f"Не удалось прочитать файл {file_path}. Пропускаем.")
+            return {}
+
+    ent_blocks = {}
+    current_ent = None
+    current_block = []
+
+    for line in content.split('\n'):
+        if line.startswith('ent-'):
+            if current_ent:
+                ent_blocks[current_ent] = '\n'.join(current_block)
+            current_ent = line.split('=')[0].strip()
+            current_block = [line]
+        elif current_ent and (line.strip().startswith('.desc') or line.strip().startswith('.suffix')):
+            current_block.append(line)
+        elif line.strip() == '':
+            if current_ent:
+                ent_blocks[current_ent] = '\n'.join(current_block)
+                current_ent = None
+                current_block = []
+        else:
+            if current_ent:
+                ent_blocks[current_ent] = '\n'.join(current_block)
+                current_ent = None
+                current_block = []
+
+    if current_ent:
+        ent_blocks[current_ent] = '\n'.join(current_block)
+
+    return ent_blocks
+
+def remove_duplicates(root_dir):
+    ftl_files = find_ftl_files(root_dir)
+    all_ents = {}
+    removed_duplicates = []
+
+    for file_path in ftl_files:
+        ent_blocks = parse_ent_blocks(file_path)
+        for ent, block in ent_blocks.items():
+            if ent not in all_ents:
+                all_ents[ent] = (file_path, block)
+
+    for file_path in ftl_files:
+        try:
+            encoding = detect_encoding(file_path)
+            with open(file_path, 'r', encoding=encoding) as file:
+                content = file.read()
+
+            ent_blocks = parse_ent_blocks(file_path)
+            for ent, block in ent_blocks.items():
+                if all_ents[ent][0] != file_path:
+                    content = content.replace(block, '')
+                    removed_duplicates.append((ent, file_path, block))
+
+            content = re.sub(r'\n{3,}', '\n\n', content)
+
+            with open(file_path, 'w', encoding=encoding) as file:
+                file.write(content)
+        except Exception as e:
+            print(f"Ошибка при обработке файла {file_path}: {str(e)}")
+
+    # Сохранение лога удаленных дубликатов
+    log_filename = f"removed_duplicates_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+    with open(log_filename, 'w', encoding='utf-8') as log_file:
+        for ent, file_path, block in removed_duplicates:
+            log_file.write(f"Удален дубликат: {ent}\n")
+            log_file.write(f"Файл: {file_path}\n")
+            log_file.write("Содержимое:\n")
+            log_file.write(block)
+            log_file.write("\n\n")
+
+    print(f"Обработка завершена. Проверено файлов: {len(ftl_files)}")
+    print(f"Лог удаленных дубликатов сохранен в файл: {log_filename}")
+
+if __name__ == "__main__":
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    main_folder = find_top_level_dir(script_dir)
+    root_dir = os.path.join(main_folder, "Resources\\Locale\\ru-RU")
+    remove_duplicates(root_dir)

Tools/ss14_ru/clean_empty.py

@@ -0,0 +1,61 @@
+import os
+import logging
+from datetime import datetime
+
+def find_top_level_dir(start_dir):
+    marker_file = 'SpaceStation14.sln'
+    current_dir = start_dir
+    while True:
+        if marker_file in os.listdir(current_dir):
+            return current_dir
+        parent_dir = os.path.dirname(current_dir)
+        if parent_dir == current_dir:
+            print(f"Не удалось найти {marker_file} начиная с {start_dir}")
+            exit(-1)
+        current_dir = parent_dir
+def setup_logging():
+    log_filename = f"cleanup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+    logging.basicConfig(filename=log_filename, level=logging.INFO,
+                        format='%(asctime)s - %(levelname)s - %(message)s')
+    console = logging.StreamHandler()
+    console.setLevel(logging.INFO)
+    logging.getLogger('').addHandler(console)
+    return log_filename
+
+def remove_empty_files_and_folders(path):
+    removed_files = 0
+    removed_folders = 0
+
+    for root, dirs, files in os.walk(path, topdown=False):
+        # Удаление пустых файлов
+        for file in files:
+            file_path = os.path.join(root, file)
+            if os.path.getsize(file_path) == 0:
+                try:
+                    os.remove(file_path)
+                    logging.info(f"Удален пустой файл: {file_path}")
+                    removed_files += 1
+                except Exception as e:
+                    logging.error(f"Ошибка при удалении файла {file_path}: {str(e)}")
+
+        # Удаление пустых папок
+        if not os.listdir(root):
+            try:
+                os.rmdir(root)
+                logging.info(f"Удалена пустая папка: {root}")
+                removed_folders += 1
+            except Exception as e:
+                logging.error(f"Ошибка при удалении папки {root}: {str(e)}")
+
+    return removed_files, removed_folders
+
+if __name__ == "__main__":
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    main_folder = find_top_level_dir(script_dir)
+    root_dir = os.path.join(main_folder, "Resources\\Locale")
+    log_file = setup_logging()
+
+    logging.info(f"Начало очистки в директории: {root_dir}")
+    files_removed, folders_removed = remove_empty_files_and_folders(root_dir)
+    logging.info(f"Очистка завершена. Удалено файлов: {files_removed}, удалено папок: {folders_removed}")
+    print(f"Лог операций сохранен в файл: {log_file}")

Tools/ss14_ru/file.py

@@ -1,8 +1,9 @@
-import typing
+import typing
 
 from fluent.syntax import ast
 from yamlmodels import YAMLElements
 import os
+import re
 
 
 class File:
@@ -42,17 +43,53 @@ class File:
 class FluentFile(File):
     def __init__(self, full_path):
         super().__init__(full_path)
+
+        self.newline_exceptions_regex = re.compile(r"^\s*[\[\]{}#%^*]")
+        self.newline_remover_tag = "%ERASE_NEWLINE%"
+        self.newline_remover_regex = re.compile(r"\n?\s*" + self.newline_remover_tag)
+
+        "%ERASE_NEWLINE%"
+
         self.full_path = full_path
 
+    def kludge(self, element):
+        return str.replace(
+            element.value,
+            self.prefixed_newline,
+            self.prefixed_newline_substitute
+        )
+
+
     def parse_data(self, file_data: typing.AnyStr):
         from fluent.syntax import FluentParser
 
-        return FluentParser().parse(file_data)
+        parsed_data = FluentParser().parse(file_data)
+
+        for body_element in parsed_data.body:
+            if not isinstance(body_element, ast.Term) and not isinstance(body_element, ast.Message):
+                continue
+
+            if not len(body_element.value.elements):
+                continue
+
+            first_element = body_element.value.elements[0]
+            if not isinstance(first_element, ast.TextElement):
+                continue
+
+            if not self.newline_exceptions_regex.match(first_element.value):
+                continue
+
+            first_element.value = f"{self.newline_remover_tag}{first_element.value}"
+
+        return parsed_data
 
     def serialize_data(self, parsed_file_data: ast.Resource):
         from fluent.syntax import FluentSerializer
 
-        return FluentSerializer(with_junk=True).serialize(parsed_file_data)
+        serialized_data = FluentSerializer(with_junk=True).serialize(parsed_file_data)
+        serialized_data = self.newline_remover_regex.sub(' ', serialized_data)
+
+        return serialized_data
 
     def read_serialized_data(self):
         return self.serialize_data(self.parse_data(self.read_data()))

Tools/ss14_ru/fluentast.py

@@ -1,4 +1,4 @@
-import typing
+import typing
 
 from fluent.syntax import ast, FluentParser, FluentSerializer
 from lokalisemodels import LokaliseKey

Tools/ss14_ru/fluentastcomparer.py

@@ -1,4 +1,4 @@
-from fluent.syntax import ast
+from fluent.syntax import ast
 from fluentast import FluentAstAbstract
 from pydash import py_
 

Tools/ss14_ru/fluentastmanager.py

@@ -1,4 +1,4 @@
-from fluent.syntax import ast
+from fluent.syntax import ast
 from fluentast import FluentAstAbstract
 
 

Tools/ss14_ru/fluentformatter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3
 
 # Форматтер, приводящий fluent-файлы (.ftl) в соответствие стайлгайду
 # path - путь к папке, содержащий форматируемые файлы. Для форматирования всего проекта, необходимо заменить значение на root_dir_path

Tools/ss14_ru/keyfinder.py

@@ -1,4 +1,4 @@
-import typing
+import typing
 import logging
 
 from pydash import py_

Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py

@@ -1,4 +1,4 @@
-from fluent.syntax import ast
+from fluent.syntax import ast
 
 from fluentast import FluentAstMessage
 from fluentastcomparer import FluentAstComparer

Tools/ss14_ru/lokalise_project.py

@@ -1,4 +1,4 @@
-import lokalise
+import lokalise
 import typing
 from lokalisemodels import LokaliseKey
 from pydash import py_

Tools/ss14_ru/lokalisemodels.py

@@ -1,4 +1,4 @@
-import typing
+import typing
 import os
 from pydash import py_
 from project import Project

Tools/ss14_ru/project.py

@@ -1,4 +1,4 @@
-import pathlib
+import pathlib
 import os
 import glob
 from file import FluentFile

Tools/ss14_ru/translation.bat

@@ -0,0 +1,9 @@
+@echo off
+
+call pip install -r requirements.txt --no-warn-script-location
+call python ./yamlextractor.py
+call python ./keyfinder.py
+call python ./clean_duplicates.py
+call python ./clean_empty.py
+
+PAUSE

Tools/ss14_ru/translation.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+# make sure to start from script dir
+if [ "$(dirname $0)" != "." ]; then
+    cd "$(dirname $0)"
+fi
+
+pip install -r requirements.txt --no-warn-script-location
+python3 ./yamlextractor.py
+python3 ./keyfinder.py
+python3 ./clean_duplicates.py
+python3 ./clean_empty.py

Tools/ss14_ru/translationsassembler.py

@@ -1,4 +1,4 @@
-import logging
+import logging
 import typing
 
 from fluent.syntax import FluentParser, FluentSerializer

Tools/ss14_ru/yamlextractor.py

@@ -1,4 +1,4 @@
-import os
+import os
 
 from fluent.syntax.parser import FluentParser
 from fluent.syntax.serializer import FluentSerializer
@@ -34,10 +34,23 @@ class YAMLExtractor:
             en_fluent_file_path = self.create_en_fluent_file(relative_parent_dir, file_name, pretty_fluent_file_serialized)
             ru_fluent_file_path = self.create_ru_fluent_file(en_fluent_file_path)
 
+    @classmethod
+    def serialize_yaml_element(cls, element):
+        parent_id = element.parent_id
+        if isinstance(parent_id, list):
+            parent_id = parent_id[0] if parent_id else 'None'
+
+        message = FluentSerializedMessage.from_yaml_element(
+            element.id, element.name,
+            FluentAstAttributeFactory.from_yaml_element(element),
+            parent_id
+        )
+
+        return message
+
+
     def get_serialized_fluent_from_yaml_elements(self, yaml_elements):
-        fluent_serialized_messages = list(
-                map(lambda el: FluentSerializedMessage.from_yaml_element(el.id, el.name, FluentAstAttributeFactory.from_yaml_element(el), el.parent_id), yaml_elements)
-            )
+        fluent_serialized_messages = list(map(YAMLExtractor.serialize_yaml_element, yaml_elements))
         fluent_exist_serialized_messages = list(filter(lambda m: m, fluent_serialized_messages))
 
         if not len(fluent_exist_serialized_messages):
@@ -49,7 +62,6 @@ class YAMLExtractor:
         en_new_dir_path = os.path.join(project.en_locale_prototypes_dir_path, relative_parent_dir)
         en_fluent_file = FluentFile(os.path.join(en_new_dir_path, f'{file_name}.ftl'))
         en_fluent_file.save_data(file_data)
-        logging.info(f'Актуализирован файл английской локали {en_fluent_file.full_path}')
 
         return en_fluent_file.full_path
 

Tools/ss14_ru/yamlmodels.py

@@ -1,4 +1,4 @@
-class YAMLEntity:
+class YAMLEntity:
     def __init__(self, id, name, description, suffix, parent_id = None):
         self.id = id
         self.name = name