From 6980ad5e12cfe9c099522d5856e530dd7f42cbec Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 27 Dec 2021 15:22:08 +0800 Subject: [PATCH 1/2] fix(ci): use pathlib do glob pattern match --- tools/ci/check_tools_files_patterns.py | 36 +++----- tools/ci/exclude_check_tools_files.txt | 7 +- tools/ci/idf_ci_utils.py | 115 ------------------------- 3 files changed, 18 insertions(+), 140 deletions(-) diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py index b6bd762de9..26337a55ec 100755 --- a/tools/ci/check_tools_files_patterns.py +++ b/tools/ci/check_tools_files_patterns.py @@ -4,30 +4,12 @@ # SPDX-License-Identifier: Apache-2.0 import argparse -import fnmatch -import glob import os import sys +from pathlib import Path import yaml -from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate - -# Monkey patch starts -# glob.glob will ignore all files starts with ``.`` -# don't ignore them here -# need to keep the same argument as glob._ishidden - - -def _ishidden(path): # pylint: disable=W0613 - return False - - -fnmatch.translate = translate - -glob.magic_check = magic_check # type: ignore -glob.magic_check_bytes = magic_check_bytes # type: ignore -glob._ishidden = _ishidden # type: ignore # pylint: disable=W0212 -# ends here +from idf_ci_utils import IDF_PATH, get_git_files def check(pattern_yml, exclude_list): @@ -37,18 +19,24 @@ def check(pattern_yml, exclude_list): if k.startswith('.pattern') and isinstance(v, list): rules_patterns_set.update(v) rules_files_set = set() + idf_path = Path(IDF_PATH) for pat in rules_patterns_set: - rules_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True)) + rules_files_set.update(idf_path.glob(pat)) exclude_patterns_set = set() - exclude_patterns_set.update([path.split('#')[0].strip() for path in open(exclude_list).readlines() if path]) + for line in open(exclude_list).readlines(): + pat = line.split('#')[0].strip() + if pat: + exclude_patterns_set.add(pat) + exclude_files_set = set() for pat in exclude_patterns_set: - exclude_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True)) + exclude_files_set.update(idf_path.glob(pat)) missing_files = set() git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True) for f in git_files: + f = Path(f) if f in rules_files_set or f in exclude_files_set: continue missing_files.add(os.path.relpath(f, IDF_PATH)) @@ -76,7 +64,7 @@ if __name__ == '__main__': if not_included_files: print('Missing Files:') for file in not_included_files: - print('\t' + file) + print('\t' + str(file)) print('Please add these files or glob patterns to ".gitlab/ci/rules.yml" and put related files under ' '".patterns-" block to trigger related tests.\n' 'Or add them to "tools/ci/exclude_check_tools_files.txt" to exclude them.') diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 51642f8e53..8eae6d5555 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -1,8 +1,13 @@ tools/ble/**/* tools/catch/**/* tools/ci/build_template_app.sh -tools/ci/check_*.{py,txt,sh} # excluded because run in default pipeline pre-check stage + +# excluded because run in default pipeline pre-check stage +tools/ci/check_*.py +tools/ci/check_*.txt +tools/ci/check_*.sh tools/ci/check_copyright_config.yaml + tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py tools/ci/ci_get_mr_info.py diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 7e36cf280c..f915509366 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -4,10 +4,8 @@ # SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # -import functools import logging import os -import re import subprocess import sys from typing import List, Optional @@ -84,119 +82,6 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]: return [os.path.join(path, f) for f in files] if full_path else files -# this function is a commit from -# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1 -# In order to solve python https://bugs.python.org/issue9584 -# glob pattern does not support brace expansion issue -def _translate(pat: str) -> str: - """Translate a shell PATTERN to a regular expression. - There is no way to quote meta-characters. - """ - i, n = 0, len(pat) - res = '' - while i < n: - c = pat[i] - i = i + 1 - if c == '*': - res = res + '.*' - elif c == '?': - res = res + '.' - elif c == '[': - j = i - if j < n and pat[j] == '!': - j = j + 1 - if j < n and pat[j] == ']': - j = j + 1 - while j < n and pat[j] != ']': - j = j + 1 - if j >= n: - res = res + '\\[' - else: - stuff = pat[i:j] - if '--' not in stuff: - stuff = stuff.replace('\\', r'\\') - else: - chunks = [] - k = i + 2 if pat[i] == '!' else i + 1 - while True: - k = pat.find('-', k, j) - if k < 0: - break - chunks.append(pat[i:k]) - i = k + 1 - k = k + 3 - chunks.append(pat[i:j]) - # Escape backslashes and hyphens for set difference (--). - # Hyphens that create ranges shouldn't be escaped. - stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') - for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) - i = j + 1 - if stuff[0] == '!': - stuff = '^' + stuff[1:] - elif stuff[0] in ('^', '['): - stuff = '\\' + stuff - res = '%s[%s]' % (res, stuff) - elif c == '{': - # Handling of brace expression: '{PATTERN,PATTERN,...}' - j = 1 - while j < n and pat[j] != '}': - j = j + 1 - if j >= n: - res = res + '\\{' - else: - stuff = pat[i:j] - i = j + 1 - - # Find indices of ',' in pattern excluding r'\,'. - # E.g. for r'a\,a,b\b,c' it will be [4, 8] - indices = [m.end() for m in re.finditer(r'[^\\],', stuff)] - - # Splitting pattern string based on ',' character. - # Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c': - # * first_part = 'a,a' - # * last_part = 'c' - # * middle_part = ['b,b'] - first_part = stuff[:indices[0] - 1].replace(r'\,', ',') - last_part = stuff[indices[-1]:].replace(r'\,', ',') - middle_parts = [ - stuff[st:en - 1].replace(r'\,', ',') - for st, en in zip(indices, indices[1:]) - ] - - # creating the regex from splitted pattern. Each part is - # recursivelly evaluated. - expanded = functools.reduce( - lambda a, b: '|'.join((a, b)), - (_translate(elem) for elem in [first_part] + middle_parts + [last_part]) - ) - res = '%s(%s)' % (res, expanded) - else: - res = res + re.escape(c) - return res - - -def translate(pat: str) -> str: - res = _translate(pat) - return r'(?s:%s)\Z' % res - - -magic_check = re.compile('([*?[{])') -magic_check_bytes = re.compile(b'([*?[{])') -# cpython github PR 6299 ends here - -# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules -# DO NOT monkey patch here, only patch where you really needs -# -# import glob -# import fnmatch -# from idf_ci_utils import magic_check, magic_check_bytes, translate -# glob.magic_check = magic_check -# glob.magic_check_bytes = magic_check_bytes -# fnmatch.translate = translate - - def is_in_directory(file_path: str, folder: str) -> bool: return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) From 70df376692df9be976dee2f226747497324ce25f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 27 Dec 2021 16:29:07 +0800 Subject: [PATCH 2/2] fix(ci): exclude patterns python files while checking tools pattern --- .gitlab/ci/rules.yml | 8 ++++++++ tools/ci/check_tools_files_patterns.py | 3 ++- tools/ci/exclude_check_tools_files.txt | 8 +++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index d1e3274ada..74be3c3654 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -27,6 +27,7 @@ - "tools/ci/python_packages/ttfw_idf/**/*" - "tools/ci/find_apps_build_apps.sh" + - "tools/ci/build_pytest_apps.py" - "tools/build_apps.py" - "tools/find_apps.py" - "tools/find_build_apps/**/*" @@ -135,6 +136,13 @@ - "tools/ci/test_reproducible_build.sh" - "tools/gen_soc_caps_kconfig/*" + - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py" + + - "tools/mkuf2.py" + - "tools/test_mkuf2/test_mkuf2.py" + + - "tools/split_paths_by_spaces.py" + .patterns-windows: &patterns-windows - "tools/windows/**/*" diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py index 26337a55ec..fd62db2971 100755 --- a/tools/ci/check_tools_files_patterns.py +++ b/tools/ci/check_tools_files_patterns.py @@ -16,8 +16,9 @@ def check(pattern_yml, exclude_list): rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader) rules_patterns_set = set() for k, v in rules_dict.items(): - if k.startswith('.pattern') and isinstance(v, list): + if k.startswith('.pattern') and k != '.patterns-python-files' and isinstance(v, list): rules_patterns_set.update(v) + rules_files_set = set() idf_path = Path(IDF_PATH) for pat in rules_patterns_set: diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 8eae6d5555..8db6df9411 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -1,13 +1,15 @@ tools/ble/**/* tools/catch/**/* tools/ci/build_template_app.sh - -# excluded because run in default pipeline pre-check stage tools/ci/check_*.py tools/ci/check_*.txt tools/ci/check_*.sh tools/ci/check_copyright_config.yaml - +tools/gdb_panic_server.py +tools/check_term.py +tools/check_python_dependencies.py +tools/python_version_checker.py +tools/generate_debug_prefix_map.py tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py tools/ci/ci_get_mr_info.py