diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 045c8ff965..248560dedb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -157,13 +157,19 @@ repos: additional_dependencies: - PyYAML == 5.3.1 - idf-build-apps~=2.0 - - id: sort-build-test-rules-ymls - name: sort .build-test-rules.yml files - entry: tools/ci/check_build_test_rules.py sort-yaml + - id: sort-yaml-files + name: sort yaml files + entry: tools/ci/sort_yaml.py language: python - files: '\.build-test-rules\.yml' + files: '\.build-test-rules\.yml$' + additional_dependencies: + - ruamel.yaml + - id: sort-yaml-test + name: sort yaml test + entry: python -m unittest tools/ci/sort_yaml.py + language: python + files: 'tools/ci/sort_yaml\.py$' additional_dependencies: - - PyYAML == 5.3.1 - ruamel.yaml - id: check-build-test-rules-path-exists name: check path in .build-test-rules.yml exists diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index e3079a3a40..4c3b6b4ae5 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -6,7 +6,6 @@ import inspect import os import re import sys -from io import StringIO from pathlib import Path from typing import Dict from typing import List @@ -345,38 +344,6 @@ def check_test_scripts( sys.exit(exit_code) -def sort_yaml(files: List[str]) -> None: - from ruamel.yaml import YAML, CommentedMap - - yaml = YAML() - yaml.indent(mapping=2, sequence=4, offset=2) - yaml.width = 4096 # avoid wrap lines - - exit_code = 0 - for f in files: - with open(f) as fr: - file_s = fr.read() - fr.seek(0) - file_d: CommentedMap = yaml.load(fr) - - sorted_yaml = CommentedMap(dict(sorted(file_d.items()))) - file_d.copy_attributes(sorted_yaml) - - with StringIO() as s: - yaml.dump(sorted_yaml, s) - - string = s.getvalue() - if string != file_s: - with open(f, 'w') as fw: - fw.write(string) - print( - f'Sorted yaml file {f}. Please take a look. sometimes the format is a bit messy' - ) - exit_code = 1 - - sys.exit(exit_code) - - def check_exist() -> None: exit_code = 0 @@ -422,9 +389,6 @@ if __name__ == '__main__': help='default build test rules config file', ) - _sort_yaml = action.add_parser('sort-yaml') - _sort_yaml.add_argument('files', nargs='+', help='all specified yaml files') - _check_exist = action.add_parser('check-exist') arg = parser.parse_args() @@ -434,9 +398,7 @@ if __name__ == '__main__': os.path.join(os.path.dirname(__file__), '..', '..') ) - if arg.action == 'sort-yaml': - sort_yaml(arg.files) - elif arg.action == 'check-exist': + if arg.action == 'check-exist': check_exist() else: check_dirs = set() diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index ead47535d0..fe9a698262 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -50,3 +50,4 @@ tools/ci/python_packages/gitlab_api.py tools/ci/python_packages/idf_http_server_test/**/* tools/ci/python_packages/idf_iperf_test_util/**/* tools/esp_prov/**/* +tools/ci/sort_yaml.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 2fce0de3c0..187a49239e 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -77,6 +77,7 @@ tools/ci/gitlab_yaml_linter.py tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh +tools/ci/sort_yaml.py tools/ci/test_autocomplete/test_autocomplete.py tools/ci/test_configure_ci_environment.sh tools/ci/test_reproducible_build.sh diff --git a/tools/ci/sort_yaml.py b/tools/ci/sort_yaml.py new file mode 100755 index 0000000000..e8eb720c7a --- /dev/null +++ b/tools/ci/sort_yaml.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +""" +Sort yaml file + +Exit non-zero if any file is modified +""" +import io +import os +import sys +import tempfile +import unittest + +from ruamel.yaml import CommentedMap +from ruamel.yaml import YAML + + +def sort_yaml(f: str) -> int: + yaml = YAML() + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.width = 4096 # avoid wrap lines`` + + exit_code = 0 + with open(f) as fr: + file_s = fr.read() + fr.seek(0) + + try: + file_d: CommentedMap = yaml.load(fr) + except Exception as e: + print(f'Failed to load yaml file {f}: {e}') + return 1 + + # sort dict keys + sorted_yaml = CommentedMap(dict(sorted(file_d.items()))) + file_d.copy_attributes(sorted_yaml) + + # sort item + for k, v in sorted_yaml.items(): + if isinstance(v, list): + sorted_yaml[k].sort() + + with io.StringIO() as s: + yaml.dump(sorted_yaml, s) + + string = s.getvalue() + if string != file_s: + with open(f, 'w') as fw: + fw.write(string) + print(f'Sorted yaml file {f}. Please take a look. sometimes the format is a bit messy') + exit_code = 1 + + return exit_code + + +class TestSortYaml(unittest.TestCase): + def test_sort_yaml(self) -> None: + _, test_yaml = tempfile.mkstemp() + with open(test_yaml, 'w') as fw: + fw.write( + '''no_runner: [] +no_env_marker: + - 1 + - 3 # foo + - 2 # bar''' + ) + + sort_yaml(fw.name) + + try: + with open(test_yaml) as fr: + self.assertEqual( + fr.read(), + '''no_env_marker: + - 1 + - 2 # bard + - 3 # foo + no_runner: []''', + ) + except AssertionError: + print(f'Please check the sorted yaml file {test_yaml}') + else: + os.remove(test_yaml) + + +if __name__ == '__main__': + ret = 0 + for _f in sys.argv[1:]: + exit_code = sort_yaml(_f) + if exit_code != 0 and ret == 0: + ret = exit_code + + sys.exit(ret)