Merge branch 'bugfix/pre-commit-windows-support' into 'master'

pre-commit: use pathlib to do glob pattern match

Closes IDF-4390

See merge request espressif/esp-idf!16547

.gitlab/ci/rules.yml

@@ -27,6 +27,7 @@
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
   - "tools/ci/find_apps_build_apps.sh"
+  - "tools/ci/build_pytest_apps.py"
   - "tools/build_apps.py"
   - "tools/find_apps.py"
   - "tools/find_build_apps/**/*"
@@ -135,6 +136,13 @@
   - "tools/ci/test_reproducible_build.sh"
 
   - "tools/gen_soc_caps_kconfig/*"
+  - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py"
+
+  - "tools/mkuf2.py"
+  - "tools/test_mkuf2/test_mkuf2.py"
+
+  - "tools/split_paths_by_spaces.py"
+
 
 .patterns-windows: &patterns-windows
   - "tools/windows/**/*"

tools/ci/check_tools_files_patterns.py

@@ -4,51 +4,40 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
-import fnmatch
-import glob
 import os
 import sys
+from pathlib import Path
 
 import yaml
-from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate
+from idf_ci_utils import IDF_PATH, get_git_files
-
-# Monkey patch starts
-# glob.glob will ignore all files starts with ``.``
-# don't ignore them here
-# need to keep the same argument as glob._ishidden
-
-
-def _ishidden(path):  # pylint: disable=W0613
-    return False
-
-
-fnmatch.translate = translate
-
-glob.magic_check = magic_check  # type: ignore
-glob.magic_check_bytes = magic_check_bytes  # type: ignore
-glob._ishidden = _ishidden  # type: ignore # pylint: disable=W0212
-# ends here
 
 
 def check(pattern_yml, exclude_list):
     rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader)
     rules_patterns_set = set()
     for k, v in rules_dict.items():
-        if k.startswith('.pattern') and isinstance(v, list):
+        if k.startswith('.pattern') and k != '.patterns-python-files' and isinstance(v, list):
             rules_patterns_set.update(v)
+
     rules_files_set = set()
+    idf_path = Path(IDF_PATH)
     for pat in rules_patterns_set:
-        rules_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
+        rules_files_set.update(idf_path.glob(pat))
 
     exclude_patterns_set = set()
-    exclude_patterns_set.update([path.split('#')[0].strip() for path in open(exclude_list).readlines() if path])
+    for line in open(exclude_list).readlines():
+        pat = line.split('#')[0].strip()
+        if pat:
+            exclude_patterns_set.add(pat)
+
     exclude_files_set = set()
     for pat in exclude_patterns_set:
-        exclude_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
+        exclude_files_set.update(idf_path.glob(pat))
 
     missing_files = set()
     git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True)
     for f in git_files:
+        f = Path(f)
         if f in rules_files_set or f in exclude_files_set:
             continue
         missing_files.add(os.path.relpath(f, IDF_PATH))
@@ -76,7 +65,7 @@ if __name__ == '__main__':
         if not_included_files:
             print('Missing Files:')
             for file in not_included_files:
-                print('\t' + file)
+                print('\t' + str(file))
             print('Please add these files or glob patterns to ".gitlab/ci/rules.yml" and put related files under '
                   '".patterns-<test_group>" block to trigger related tests.\n'
                   'Or add them to "tools/ci/exclude_check_tools_files.txt" to exclude them.')

tools/ci/exclude_check_tools_files.txt

@@ -1,8 +1,15 @@
 tools/ble/**/*
 tools/catch/**/*
 tools/ci/build_template_app.sh
-tools/ci/check_*.{py,txt,sh}  # excluded because run in default pipeline pre-check stage
+tools/ci/check_*.py
+tools/ci/check_*.txt
+tools/ci/check_*.sh
 tools/ci/check_copyright_config.yaml
+tools/gdb_panic_server.py
+tools/check_term.py
+tools/check_python_dependencies.py
+tools/python_version_checker.py
+tools/generate_debug_prefix_map.py
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py
 tools/ci/ci_get_mr_info.py

tools/ci/idf_ci_utils.py

@@ -4,10 +4,8 @@
 # SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
-import functools
 import logging
 import os
-import re
 import subprocess
 import sys
 from typing import List, Optional
@@ -84,119 +82,6 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]:
     return [os.path.join(path, f) for f in files] if full_path else files
 
 
-# this function is a commit from
-# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1
-# In order to solve python https://bugs.python.org/issue9584
-# glob pattern does not support brace expansion issue
-def _translate(pat: str) -> str:
-    """Translate a shell PATTERN to a regular expression.
-    There is no way to quote meta-characters.
-    """
-    i, n = 0, len(pat)
-    res = ''
-    while i < n:
-        c = pat[i]
-        i = i + 1
-        if c == '*':
-            res = res + '.*'
-        elif c == '?':
-            res = res + '.'
-        elif c == '[':
-            j = i
-            if j < n and pat[j] == '!':
-                j = j + 1
-            if j < n and pat[j] == ']':
-                j = j + 1
-            while j < n and pat[j] != ']':
-                j = j + 1
-            if j >= n:
-                res = res + '\\['
-            else:
-                stuff = pat[i:j]
-                if '--' not in stuff:
-                    stuff = stuff.replace('\\', r'\\')
-                else:
-                    chunks = []
-                    k = i + 2 if pat[i] == '!' else i + 1
-                    while True:
-                        k = pat.find('-', k, j)
-                        if k < 0:
-                            break
-                        chunks.append(pat[i:k])
-                        i = k + 1
-                        k = k + 3
-                    chunks.append(pat[i:j])
-                    # Escape backslashes and hyphens for set difference (--).
-                    # Hyphens that create ranges shouldn't be escaped.
-                    stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
-                                     for s in chunks)
-                # Escape set operations (&&, ~~ and ||).
-                stuff = re.sub(r'([&~|])', r'\\\1', stuff)
-                i = j + 1
-                if stuff[0] == '!':
-                    stuff = '^' + stuff[1:]
-                elif stuff[0] in ('^', '['):
-                    stuff = '\\' + stuff
-                res = '%s[%s]' % (res, stuff)
-        elif c == '{':
-            # Handling of brace expression: '{PATTERN,PATTERN,...}'
-            j = 1
-            while j < n and pat[j] != '}':
-                j = j + 1
-            if j >= n:
-                res = res + '\\{'
-            else:
-                stuff = pat[i:j]
-                i = j + 1
-
-                # Find indices of ',' in pattern excluding r'\,'.
-                # E.g. for r'a\,a,b\b,c' it will be [4, 8]
-                indices = [m.end() for m in re.finditer(r'[^\\],', stuff)]
-
-                # Splitting pattern string based on ',' character.
-                # Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c':
-                # * first_part = 'a,a'
-                # * last_part = 'c'
-                # * middle_part = ['b,b']
-                first_part = stuff[:indices[0] - 1].replace(r'\,', ',')
-                last_part = stuff[indices[-1]:].replace(r'\,', ',')
-                middle_parts = [
-                    stuff[st:en - 1].replace(r'\,', ',')
-                    for st, en in zip(indices, indices[1:])
-                ]
-
-                # creating the regex from splitted pattern. Each part is
-                # recursivelly evaluated.
-                expanded = functools.reduce(
-                    lambda a, b: '|'.join((a, b)),
-                    (_translate(elem) for elem in [first_part] + middle_parts + [last_part])
-                )
-                res = '%s(%s)' % (res, expanded)
-        else:
-            res = res + re.escape(c)
-    return res
-
-
-def translate(pat: str) -> str:
-    res = _translate(pat)
-    return r'(?s:%s)\Z' % res
-
-
-magic_check = re.compile('([*?[{])')
-magic_check_bytes = re.compile(b'([*?[{])')
-# cpython github PR 6299 ends here
-
-# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules
-# DO NOT monkey patch here, only patch where you really needs
-#
-# import glob
-# import fnmatch
-# from idf_ci_utils import magic_check, magic_check_bytes, translate
-# glob.magic_check = magic_check
-# glob.magic_check_bytes = magic_check_bytes
-# fnmatch.translate = translate
-
-
 def is_in_directory(file_path: str, folder: str) -> bool:
     return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)