ci(rules): add check_tools_file_patterns.py

this file is used to check if all files under `tools` folder are
recorded in patterns or in exclude list

.gitlab/ci/pre_check.yml

@@ -147,3 +147,12 @@ check_version_tag:
     - .rules:tag:release-no_label
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
+
+check_tools_file_patterns:
+  extends: .pre_check_job_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  variables:
+    PYTHON_VER: 3.7.7
+  script:
+    - python tools/ci/check_tools_files_patterns.py
+  allow_failure: true

tools/ci/check_tools_files_patterns.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#
+# Copyright 2021 Espressif Systems (Shanghai) CO LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import fnmatch
+import glob
+import os
+import sys
+
+import yaml
+from idf_ci_utils import IDF_PATH, get_git_files, magic_check, magic_check_bytes, translate
+
+# Monkey patch starts
+
+# glob.glob will ignore all files starts with ``.``
+# don't ignore them here
+# need to keep the same argument as glob._ishidden
+
+
+def _ishidden(path):  # pylint: disable=W0613
+    return False
+
+
+fnmatch.translate = translate
+
+glob.magic_check = magic_check
+glob.magic_check_bytes = magic_check_bytes
+glob._ishidden = _ishidden  # pylint: disable=W0212
+# ends here
+
+
+def check(pattern_yml, exclude_list):
+    rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader)
+    rules_patterns_set = set()
+    for k, v in rules_dict.items():
+        if k.startswith('.pattern') and isinstance(v, list):
+            rules_patterns_set.update(v)
+    rules_files_set = set()
+    for pat in rules_patterns_set:
+        rules_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
+
+    exclude_patterns_set = set()
+    exclude_patterns_set.update([path.split('#')[0].strip() for path in open(exclude_list).readlines() if path])
+    exclude_files_set = set()
+    for pat in exclude_patterns_set:
+        exclude_files_set.update(glob.glob(os.path.join(IDF_PATH, pat), recursive=True))
+
+    missing_files = set()
+    git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True)
+    for f in git_files:
+        if f in rules_files_set or f in exclude_files_set:
+            continue
+        missing_files.add(os.path.relpath(f, IDF_PATH))
+
+    return missing_files, rules_patterns_set.intersection(exclude_patterns_set)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='check if all tools files are in rules patterns or exclude list')
+    parser.add_argument('-c', '--pattern-yml',
+                        default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
+                        help='yml file path included file patterns')
+    parser.add_argument('-e', '--exclude-list',
+                        default=os.path.join(IDF_PATH, 'tools', 'ci', 'exclude_check_tools_files.txt'),
+                        help='exclude list path')
+
+    args = parser.parse_args()
+
+    res = 0
+    not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list)
+    if not_included_files:
+        print('Missing Files: (please add to tools/ci/exclude_check_tools_files.txt')
+        for f in not_included_files:
+            print(f)
+        res = 1
+    if dup_patterns:
+        print('Duplicated Patterns: (please check .gitlab/ci/rules.yml and tools/ci/exclude_check_tools_files.txt')
+        for pat in dup_patterns:
+            print(pat)
+        res = 1
+
+    sys.exit(res)

tools/ci/exclude_check_tools_files.txt

@@ -0,0 +1,32 @@
+tools/ble/**/*
+tools/catch/**/*
+tools/ci/build_template_app.sh
+tools/ci/check_*.{py,txt,sh}  # excluded because run in default pipeline pre-check stage
+tools/ci/checkout_project_ref.py
+tools/ci/ci_fetch_submodule.py
+tools/ci/ci_get_mr_info.py
+tools/ci/configure_ci_environment.sh
+tools/ci/deploy_docs.py
+tools/ci/envsubst.py
+tools/ci/*exclude*.txt
+tools/ci/executable-list.txt
+tools/ci/fix_empty_prototypes.sh
+tools/ci/get-full-sources.sh
+tools/ci/idf_ci_utils.py
+tools/ci/mirror-submodule-update.sh
+tools/ci/multirun_with_pyenv.sh
+tools/ci/normalize_clangtidy_path.py
+tools/ci/push_to_github.sh
+tools/ci/python_packages/wifi_tools.py
+tools/ci/setup_python.sh
+tools/ci/utils.sh
+tools/eclipse-code-style.xml
+tools/format-minimal.sh
+tools/format.sh
+tools/gen_esp_err_to_name.py
+tools/kconfig/**/*
+tools/set-submodules-to-github.sh
+tools/templates/sample_component/CMakeLists.txt
+tools/templates/sample_component/include/main.h
+tools/templates/sample_component/main.c
+tools/toolchain_versions.mk

tools/ci/executable-list.txt

@@ -46,6 +46,7 @@ tools/ci/check_idf_version.sh
 tools/ci/check_kconfigs.py
 tools/ci/check_readme_links.py
 tools/ci/check_rom_apis.sh
+tools/ci/check_tools_files_patterns.py
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
 tools/ci/deploy_docs.py

tools/ci/idf_ci_utils.py

@@ -15,13 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
+import functools
 import logging
 import os
+import re
 import subprocess
 import sys
 
-IDF_PATH = os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
+IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
 
 
 def get_submodule_dirs(full_path=False):  # type: (bool) -> list
@@ -41,7 +42,7 @@ def get_submodule_dirs(full_path=False):  # type: (bool) -> list
                 dirs.append(os.path.join(IDF_PATH, path))
             else:
                 dirs.append(path)
-    except Exception as e:
+    except Exception as e:  # pylint: disable=W0703
         logging.warning(str(e))
 
     return dirs
@@ -67,5 +68,132 @@ def is_executable(full_path):  # type: (str) -> bool
     """
     if sys.platform == 'win32':
         return _check_git_filemode(full_path)
-    else:
-        return os.access(full_path, os.X_OK)
+    return os.access(full_path, os.X_OK)
+
+
+def get_git_files(path=IDF_PATH, full_path=False):  # type: (str, bool) -> list[str]
+    """
+    Get the result of git ls-files
+    :param path: path to run git ls-files
+    :param full_path: return full path if set to True
+    :return: list of file paths
+    """
+    try:
+        files = subprocess.check_output(['git', 'ls-files'], cwd=path).decode('utf8').strip().split('\n')
+    except Exception as e:  # pylint: disable=W0703
+        logging.warning(str(e))
+        files = []
+    return [os.path.join(path, f) for f in files] if full_path else files
+
+
+# this function is a commit from
+# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1
+# In order to solve python https://bugs.python.org/issue9584
+# glob pattern does not support brace expansion issue
+def _translate(pat):
+    """Translate a shell PATTERN to a regular expression.
+    There is no way to quote meta-characters.
+    """
+    i, n = 0, len(pat)
+    res = ''
+    while i < n:
+        c = pat[i]
+        i = i + 1
+        if c == '*':
+            res = res + '.*'
+        elif c == '?':
+            res = res + '.'
+        elif c == '[':
+            j = i
+            if j < n and pat[j] == '!':
+                j = j + 1
+            if j < n and pat[j] == ']':
+                j = j + 1
+            while j < n and pat[j] != ']':
+                j = j + 1
+            if j >= n:
+                res = res + '\\['
+            else:
+                stuff = pat[i:j]
+                if '--' not in stuff:
+                    stuff = stuff.replace('\\', r'\\')
+                else:
+                    chunks = []
+                    k = i + 2 if pat[i] == '!' else i + 1
+                    while True:
+                        k = pat.find('-', k, j)
+                        if k < 0:
+                            break
+                        chunks.append(pat[i:k])
+                        i = k + 1
+                        k = k + 3
+                    chunks.append(pat[i:j])
+                    # Escape backslashes and hyphens for set difference (--).
+                    # Hyphens that create ranges shouldn't be escaped.
+                    stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
+                                     for s in chunks)
+                # Escape set operations (&&, ~~ and ||).
+                stuff = re.sub(r'([&~|])', r'\\\1', stuff)
+                i = j + 1
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] in ('^', '['):
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
+        elif c == '{':
+            # Handling of brace expression: '{PATTERN,PATTERN,...}'
+            j = 1
+            while j < n and pat[j] != '}':
+                j = j + 1
+            if j >= n:
+                res = res + '\\{'
+            else:
+                stuff = pat[i:j]
+                i = j + 1
+
+                # Find indices of ',' in pattern excluding r'\,'.
+                # E.g. for r'a\,a,b\b,c' it will be [4, 8]
+                indices = [m.end() for m in re.finditer(r'[^\\],', stuff)]
+
+                # Splitting pattern string based on ',' character.
+                # Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c':
+                # * first_part = 'a,a'
+                # * last_part = 'c'
+                # * middle_part = ['b,b']
+                first_part = stuff[:indices[0] - 1].replace(r'\,', ',')
+                last_part = stuff[indices[-1]:].replace(r'\,', ',')
+                middle_parts = [
+                    stuff[st:en - 1].replace(r'\,', ',')
+                    for st, en in zip(indices, indices[1:])
+                ]
+
+                # creating the regex from splitted pattern. Each part is
+                # recursivelly evaluated.
+                expanded = functools.reduce(
+                    lambda a, b: '|'.join((a, b)),
+                    (_translate(elem) for elem in [first_part] + middle_parts + [last_part])
+                )
+                res = '%s(%s)' % (res, expanded)
+        else:
+            res = res + re.escape(c)
+    return res
+
+
+def translate(pat):
+    res = _translate(pat)
+    return r'(?s:%s)\Z' % res
+
+
+magic_check = re.compile('([*?[{])')
+magic_check_bytes = re.compile(b'([*?[{])')
+# cpython github PR 6299 ends here
+
+# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules
+# DO NOT monkey patch here, only patch where you really needs
+#
+# import glob
+# import fnmatch
+# from idf_ci_utils import magic_check, magic_check_bytes, translate
+# glob.magic_check = magic_check
+# glob.magic_check_bytes = magic_check_bytes
+# fnmatch.translate = translate