CI: Add pre-commit for esp-idf project.

add tools/ci/python_packages/idf_ci.py for some util functions used in
ci and needs multi-os solution

.gitlab/CODEOWNERS

@@ -48,6 +48,7 @@
 
 /.*                                   @esp-idf-codeowners/tools
 /.gitlab-ci.yml                       @esp-idf-codeowners/ci
+/.pre-commit-config.yaml              @esp-idf-codeowners/ci
 /.readthedocs.yml                     @esp-idf-codeowners/docs
 /CMakeLists.txt                       @esp-idf-codeowners/build-config
 /Kconfig                              @esp-idf-codeowners/build-config

.pre-commit-config.yaml

@@ -0,0 +1,60 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: '.+\.(md|rst)'
+      - id: end-of-file-fixer
+      - id: check-executables-have-shebangs
+      - id: file-contents-sorter
+        files: 'tools/ci/executable-list.txt'
+      - id: mixed-line-ending
+        args: ['-f=lf']
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8
+        args: ['--config=.flake8', '--tee', '--benchmark']
+  - repo: local
+    hooks:
+      - id: check-executables
+        name: Check File Permissions
+        entry: tools/ci/check_executables.py --action executables
+        language: python
+        types: [executable]
+        exclude: '\.pre-commit/.+'
+      - id: check-executable-list
+        name: Validate executable-list.txt
+        entry: tools/ci/check_executables.py --action list
+        language: python
+        pass_filenames: false
+        always_run: true
+      - id: check-kconfigs
+        name: Validate Kconfig files
+        entry: tools/ci/check_kconfigs.py --exclude-submodules
+        language: python
+        pass_filenames: false
+        always_run: true
+      - id: check-deprecated-kconfigs-options
+        name: Check if any Kconfig Options Deprecated
+        entry: tools/ci/check_deprecated_kconfigs.py --exclude-submodules
+        language: python
+        pass_filenames: false
+        always_run: true
+      - id: cmake-lint
+        name: Check CMake Files Format
+        entry: cmakelint --linelength=120 --spaces=4
+        language: python
+        additional_dependencies:
+          - https://github.com/richq/cmake-lint/archive/058c6c0ed2536.zip
+        files: 'CMakeLists.txt$|\.cmake$'
+        exclude: '\/third_party\/'
+      - id: check-codeowners
+        name: Validate Codeowner File
+        entry: tools/ci/check_codeowners.py ci-check
+        language: python
+        files: '\.gitlab/CODEOWNERS'
+        pass_filenames: false

CONTRIBUTING.rst

@@ -17,6 +17,8 @@ Before sending us a Pull Request, please consider this list of points:
 
 * Does any new code conform to the esp-idf :doc:`Style Guide <style-guide>`?
 
+* Have you installed the pre-commit hook for esp-idf? (please refer to https://pre-commit.com/#installation)
+
 * Does the code documentation follow requirements in :doc:`documenting-code`?
 
 * Is the code adequately commented for people to understand how it is structured?

tools/ci/check-executable.sh

@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-# This script finds executable files in the repository, excluding some directories,
-# then prints the list of all files which are not in executable-list.txt.
-# Returns with error if this list is non-empty.
-# Also checks if executable-list.txt is sorted and has no duplicates.
-
-set -o errexit # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-set -o nounset # Exit if variable not set.
-
-
-cd $IDF_PATH
-
-in_list=tools/ci/executable-list.txt
-tmp_list=$(mktemp)
-out_list=$(mktemp)
-
-# build exclude pattern like '-o -path ./components/component/submodule' for each submodule
-submodule_excludes=$(git config --file .gitmodules --get-regexp path | awk '{ print "-o -path ./" $2 }')
-
-# figure out which flag to use when searching for executable files
-if [ "$(uname -s)" == "Darwin" ]; then
-    perm_flag="-perm +111"
-else
-    perm_flag="-executable"
-fi
-
-find . -type d \( \
-            -path ./.git \
-            -o -name build \
-            -o -name builds \
-            $submodule_excludes \
-        \) -prune -o -type f $perm_flag -print \
-    | sed "s|^\./||" > $tmp_list
-
-# this looks for lines present in tmp_list but not in executable-list.txt
-comm -13 <(cat $in_list | sed -n "/^#/!p" | sort) <(sort $tmp_list) > $out_list
-
-ret=0
-if [ -s $out_list ]; then
-    ret=1
-    echo "Error: the following file(s) have executable flag set:"
-    echo ""
-    cat $out_list
-    echo ""
-    echo "If any files need to be executable (usually, scripts), add them to tools/ci/executable-list.txt"
-    echo "Make the rest of the files non-executable using 'chmod -x <filename>'."
-    echo "On Windows, use 'git update-index --chmod=-x filename' instead."
-    echo ""
-fi
-
-if ! diff <(cat $in_list | sed -n "/^#/!p" | sort | uniq) $in_list; then
-    echo "$in_list is not sorted or has duplicate entries"
-    ret=2
-fi
-
-for filename in $(cat $in_list | sed -n "/^#/!p"); do
-    if [ ! -f "$filename" ]; then
-        echo "Warning: file '$filename' is present in '$in_list', but does not exist"
-    fi
-done
-
-rm $tmp_list
-rm $out_list
-
-exit $ret

tools/ci/check-line-endings.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-if ! [ -z "$1" ]; then
-    cd "$1"
-fi
-
-echo "Checking for Windows line endings in `pwd`"
-
-if git ls-tree --name-only -r HEAD | xargs file -N | grep CRLF; then
-    echo "Some files have CRLF (Windows-style) line endings. Please convert to LF (Unix-style). git can be configured to do this automatically."
-    exit 1
-fi
-
-exit 0

tools/ci/check_codeowners.py

@@ -22,8 +22,9 @@ import re
 import subprocess
 import sys
 
+from idf_ci_utils import IDF_PATH
 
-CODEOWNERS_PATH = os.path.join(os.path.dirname(__file__), "..", ".gitlab", "CODEOWNERS")
+CODEOWNERS_PATH = os.path.join(IDF_PATH, ".gitlab", "CODEOWNERS")
 CODEOWNER_GROUP_PREFIX = "@esp-idf-codeowners/"
 
 
@@ -31,9 +32,8 @@ def get_all_files():
     """
     Get list of all file paths in the repository.
     """
-    idf_root = os.path.join(os.path.dirname(__file__), "..")
     # only split on newlines, since file names may contain spaces
-    return subprocess.check_output(["git", "ls-files"], cwd=idf_root).decode("utf-8").strip().split('\n')
+    return subprocess.check_output(["git", "ls-files"], cwd=IDF_PATH).decode("utf-8").strip().split('\n')
 
 
 def pattern_to_regex(pattern):
@@ -121,7 +121,7 @@ def action_ci_check(args):
     errors = []
 
     def add_error(msg):
-        errors.append("Error at CODEOWNERS:{}: {}".format(line_no, msg))
+        errors.append("{}:{}: {}".format(CODEOWNERS_PATH, line_no, msg))
 
     all_files = get_all_files()
     prev_path_pattern = ""

tools/ci/check_deprecated_kconfigs.py

@@ -20,6 +20,7 @@ import argparse
 import os
 import sys
 from io import open
+from idf_ci_utils import get_submodule_dirs
 
 # FILES_TO_CHECK used as "startswith" pattern to match sdkconfig.defaults variants
 FILES_TO_CHECK = ('sdkconfig.ci', 'sdkconfig.defaults')
@@ -52,10 +53,11 @@ def main():
 
     parser = argparse.ArgumentParser(description='Kconfig options checker')
     parser.add_argument('--directory', '-d', help='Path to directory to check recursively  '
-                        '(for example $IDF_PATH)',
+                                                  '(for example $IDF_PATH)',
                         type=_valid_directory,
                         required=default_path is None,
                         default=default_path)
+    parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules')
     args = parser.parse_args()
 
     # IGNORE_DIRS makes sense when the required directory is IDF_PATH
@@ -64,7 +66,12 @@ def main():
     ignores = 0
     files_to_check = []
     deprecated_options = set()
-    errors = []
+    ret = 0
+
+    ignore_dirs = IGNORE_DIRS
+    if args.exclude_submodules:
+        for submodule in get_submodule_dirs():
+            ignore_dirs = ignore_dirs + tuple(submodule)
 
     for root, dirnames, filenames in os.walk(args.directory):
         for filename in filenames:
@@ -72,7 +79,7 @@ def main():
             path_in_idf = os.path.relpath(full_path, args.directory)
 
             if filename.startswith(FILES_TO_CHECK):
-                if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS):
+                if check_ignore_dirs and path_in_idf.startswith(ignore_dirs):
                     print('{}: Ignored'.format(path_in_idf))
                     ignores += 1
                     continue
@@ -84,16 +91,13 @@ def main():
         used_options = _parse_path(path, '=')
         used_deprecated_options = deprecated_options & used_options
         if len(used_deprecated_options) > 0:
-            errors.append('{}: The following options are deprecated: {}'.format(path,
-                                                                                ', '.join(used_deprecated_options)))
+            print('{}: The following options are deprecated: {}'.format(path, ', '.join(used_deprecated_options)))
+            ret = 1
 
     if ignores > 0:
         print('{} files have been ignored.'.format(ignores))
-
-    if len(errors) > 0:
-        print('\n\n'.join(errors))
-        sys.exit(1)
+    return ret
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

tools/ci/check_executables.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+from sys import exit
+
+
+def _strip_each_item(iterable):
+    res = []
+    for item in iterable:
+        if item:
+            res.append(item.strip())
+    return res
+
+
+IDF_PATH = os.getenv('IDF_PATH', os.getcwd())
+EXECUTABLE_LIST_FN = os.path.join(IDF_PATH, 'tools/ci/executable-list.txt')
+known_executables = _strip_each_item(open(EXECUTABLE_LIST_FN).readlines())
+
+
+def check_executable_list():
+    ret = 0
+    for index, fn in enumerate(known_executables):
+        if not os.path.exists(os.path.join(IDF_PATH, fn)):
+            print('{}:{} {} not exists. Please remove it manually'.format(EXECUTABLE_LIST_FN, index + 1, fn))
+            ret = 1
+    return ret
+
+
+def check_executables(files):
+    ret = 0
+    for fn in files:
+        if fn not in known_executables:
+            print('"{}" is not in {}'.format(fn, EXECUTABLE_LIST_FN))
+            ret = 1
+    return ret
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--action', choices=['executables', 'list'], required=True,
+                        help='if "executables", pass all your executables to see if it\'s in the list.'
+                             'if "list", check if all items on your list exist')
+    parser.add_argument('filenames', nargs='*', help='Filenames to check.')
+    args = parser.parse_args()
+
+    if args.action == 'executables':
+        ret = check_executables(args.filenames)
+    elif args.action == 'list':
+        ret = check_executable_list()
+    else:
+        raise ValueError
+
+    return ret
+
+
+if __name__ == '__main__':
+    exit(main())

tools/ci/check_kconfigs.py

@@ -21,6 +21,7 @@ import sys
 import re
 import argparse
 from io import open
+from idf_ci_utils import get_submodule_dirs
 
 # regular expression for matching Kconfig files
 RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$'
@@ -377,16 +378,22 @@ def main():
     parser.add_argument('--verbose', '-v', help='Print more information (useful for debugging)',
                         action='store_true', default=False)
     parser.add_argument('--directory', '-d', help='Path to directory where Kconfigs should be recursively checked '
-                        '(for example $IDF_PATH)',
+                                                  '(for example $IDF_PATH)',
                         type=valid_directory,
                         required=default_path is None,
                         default=default_path)
+    parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules')
     args = parser.parse_args()
 
-    success_couter = 0
+    success_counter = 0
     ignore_counter = 0
     failure = False
 
+    ignore_dirs = IGNORE_DIRS
+    if args.exclude_submodules:
+        for submodule in get_submodule_dirs():
+            ignore_dirs = ignore_dirs + tuple(submodule)
+
     # IGNORE_DIRS makes sense when the required directory is IDF_PATH
     check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path)
 
@@ -395,7 +402,7 @@ def main():
             full_path = os.path.join(root, filename)
             path_in_idf = os.path.relpath(full_path, args.directory)
             if re.search(RE_KCONFIG, filename):
-                if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS):
+                if check_ignore_dirs and path_in_idf.startswith(ignore_dirs):
                     print('{}: Ignored'.format(path_in_idf))
                     ignore_counter += 1
                     continue
@@ -425,9 +432,9 @@ def main():
                           'for solving all issues'.format(path_in_idf + OUTPUT_SUFFIX))
                     print('Please fix the errors and run {} for checking the correctness of '
                           'Kconfigs.'.format(os.path.relpath(os.path.abspath(__file__), args.directory)))
-                    sys.exit(1)
+                    return 1
                 else:
-                    success_couter += 1
+                    success_counter += 1
                     print('{}: OK'.format(path_in_idf))
                     try:
                         os.remove(suggestions_full_path)
@@ -440,10 +447,10 @@ def main():
 
     if ignore_counter > 0:
         print('{} files have been ignored.'.format(ignore_counter))
-
-    if success_couter > 0:
-        print('{} files have been successfully checked.'.format(success_couter))
+    if success_counter > 0:
+        print('{} files have been successfully checked.'.format(success_counter))
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

tools/ci/ci_get_latest_mr_iid.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-#
-# internal use only for CI
-# get latest MR IID by source branch
-
-import argparse
-import os
-
-from gitlab_api import Gitlab
-
-
-def get_MR_IID_by_source_branch(source_branch):
-    if not source_branch:
-        return ''
-    gl = Gitlab(os.getenv('CI_PROJECT_ID'))
-    if not gl.project:
-        return ''
-    mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
-    if mrs:
-        mr = mrs[0]  # one source branch can only have one opened MR at one moment
-        return mr.iid
-    return ''
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get the latest MR IID by source branch, if not found, return empty string')
-    parser.add_argument('source_branch', nargs='?', help='source_branch')  # won't fail if it's empty
-
-    args = parser.parse_args()
-
-    print(get_MR_IID_by_source_branch(args.source_branch))

tools/ci/ci_get_mr_info.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+#
+# internal use only for CI
+# get latest MR information by source branch
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import os
+import subprocess
+
+from gitlab_api import Gitlab
+
+
+def _get_mr_obj(source_branch):
+    if not source_branch:
+        return None
+    gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
+    if not gl.project:
+        return None
+    mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
+    if mrs:
+        return mrs[0]  # one source branch can only have one opened MR at one moment
+    else:
+        return None
+
+
+def get_mr_iid(source_branch):  # type: (str) -> str
+    mr = _get_mr_obj(source_branch)
+    if not mr:
+        return ''
+    else:
+        return str(mr.iid)
+
+
+def get_mr_changed_files(source_branch):
+    mr = _get_mr_obj(source_branch)
+    if not mr:
+        return ''
+
+    return subprocess.check_output(['git', 'diff', '--name-only',
+                                    'origin/{}...origin/{}'.format(mr.target_branch, source_branch)]).decode('utf8')
+
+
+def get_mr_commits(source_branch):
+    mr = _get_mr_obj(source_branch)
+    if not mr:
+        return ''
+    return '\n'.join([commit.id for commit in mr.commits()])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
+    actions = parser.add_subparsers(dest='action', help='info type')
+
+    common_args = argparse.ArgumentParser(add_help=False)
+    common_args.add_argument('src_branch', nargs='?', help='source branch')
+
+    actions.add_parser('id', parents=[common_args])
+    actions.add_parser('files', parents=[common_args])
+    actions.add_parser('commits', parents=[common_args])
+
+    args = parser.parse_args()
+
+    if args.action == 'id':
+        print(get_mr_iid(args.src_branch))
+    elif args.action == 'files':
+        print(get_mr_changed_files(args.src_branch))
+    elif args.action == 'commits':
+        print(get_mr_commits(args.src_branch))
+    else:
+        raise NotImplementedError('not possible to get here')

tools/ci/config/build.yml

@@ -432,9 +432,11 @@ code_quality_check:
     - .rules:trigger
   allow_failure: true
   script:
-    - export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs`
+    - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH})
+    - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',')
     # test if this branch have merge request, if not, exit 0
     - test -n "$CI_MERGE_REQUEST_IID" || exit 0
+    - test -n "$CI_MR_COMMITS" || exit 0
     - sonar-scanner
       -Dsonar.analysis.mode=preview
       -Dsonar.host.url=$SONAR_HOST_URL
@@ -445,12 +447,12 @@ code_quality_check:
       -Dsonar.projectBaseDir=$CI_PROJECT_DIR
       -Dsonar.exclusions=$EXCLUSIONS
       -Dsonar.gitlab.project_id=$CI_PROJECT_ID
-      -Dsonar.gitlab.commit_sha=$(git log --pretty=format:%H origin/master..origin/$CI_COMMIT_REF_NAME | tr '\n' ',')
+      -Dsonar.gitlab.commit_sha=CI_MR_COMMITS
       -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
       -Dsonar.cxx.includeDirectories=components,/usr/include
       -Dsonar.python.pylint_config=.pylintrc
-      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
+      -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID
       -Dsonar.gitlab.merge_request_discussion=true
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
 

tools/ci/config/pre_check.yml

@@ -15,15 +15,27 @@
     - .pre_check_base_template
     - .before_script_lesser
 
-check_line_endings:
+.check_pre_commit_template:
   extends: .pre_check_job_template
-  script:
-    - tools/ci/check-line-endings.sh ${IDF_PATH}
+  stage: pre_check
+  image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
+  before_script:
+    - source tools/ci/utils.sh
+    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
 
-check_permissions:
-  extends: .pre_check_job_template
+check_pre_commit_master_release:
+  extends:
+    - .check_pre_commit_template
+    - .rules:protected
   script:
-    - tools/ci/check-executable.sh
+    - git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA | xargs pre-commit run --files
+
+check_pre_commit_MR:
+  extends:
+    - .check_pre_commit_template
+    - .rules:dev
+  script:
+    - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_COMMIT_BRANCH} | xargs pre-commit run --files
 
 check_docs_lang_sync:
   extends: .pre_check_job_template
@@ -79,18 +91,8 @@ check_kconfigs:
       - tools/*/*/*/Kconfig*.new
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
-    - ${IDF_PATH}/tools/check_kconfigs.py
-
-check_deprecated_kconfig_options:
-  extends: .pre_check_job_template_with_filter
-  script:
-    - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
-
-check_cmake_style:
-  extends: .pre_check_job_template
-  script:
-    tools/cmake/run_cmake_lint.sh
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+    - ${IDF_PATH}/tools/ci/check_kconfigs.py
 
 check_wifi_lib_md5:
   extends: .pre_check_base_template
@@ -183,11 +185,6 @@ clang_tidy_check_all:
     BOT_NEEDS_TRIGGER_BY_NAME: 1
     BOT_LABEL_STATIC_ANALYSIS_ALL: 1
 
-check_codeowners:
-  extends: .pre_check_job_template
-  script:
-    - tools/codeowners.py ci-check
-
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:

tools/ci/executable-list.txt

@@ -32,18 +32,18 @@ examples/system/ota/otatool/otatool_example.sh
 install.fish
 install.sh
 tools/build_apps.py
-tools/check_kconfigs.py
 tools/check_python_dependencies.py
 tools/ci/apply_bot_filter.py
 tools/ci/build_template_app.sh
-tools/ci/check-executable.sh
-tools/ci/check-line-endings.sh
 tools/ci/check_build_warnings.py
 tools/ci/check_callgraph.py
+tools/ci/check_codeowners.py
 tools/ci/check_deprecated_kconfigs.py
 tools/ci/check_examples_cmake_make.py
 tools/ci/check_examples_rom_header.sh
+tools/ci/check_executables.py
 tools/ci/check_idf_version.sh
+tools/ci/check_kconfigs.py
 tools/ci/check_readme_links.py
 tools/ci/check_rom_apis.sh
 tools/ci/check_ut_cmake_make.sh
@@ -60,11 +60,10 @@ tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
 tools/ci/test_build_system.sh
 tools/ci/test_build_system_cmake.sh
+tools/ci/test_check_kconfigs.py
 tools/ci/test_configure_ci_environment.sh
 tools/ci/utils.sh
 tools/cmake/convert_to_cmake.py
-tools/cmake/run_cmake_lint.sh
-tools/codeowners.py
 tools/docker/entrypoint.sh
 tools/docker/hooks/build
 tools/esp_app_trace/logtrace_proc.py
@@ -93,7 +92,6 @@ tools/ldgen/test/test_generation.py
 tools/mass_mfg/mfg_gen.py
 tools/mkdfu.py
 tools/set-submodules-to-github.sh
-tools/test_check_kconfigs.py
 tools/test_idf_monitor/run_test_idf_monitor.py
 tools/test_idf_py/test_idf_py.py
 tools/test_idf_size/test.sh

tools/ci/idf_ci_utils.py

@@ -0,0 +1,43 @@
+# internal use only for CI
+# some CI related util functions
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+import subprocess
+
+IDF_PATH = os.getenv('IDF_PATH', os.getcwd())
+
+
+def get_submodule_dirs():  # type: () -> list
+    """
+    To avoid issue could be introduced by multi-os or additional dependency,
+    we use python and git to get this output
+    :return: List of submodule dirs
+    """
+    dirs = []
+    try:
+        lines = subprocess.check_output(
+            ['git', 'config', '--file', os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')),
+             '--get-regexp', 'path']).decode('utf8').strip().split('\n')
+        for line in lines:
+            _, path = line.split(' ')
+            dirs.append(path)
+    except Exception as e:
+        logging.warning(str(e))
+
+    return dirs

tools/cmake/run_cmake_lint.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-#
-# Run cmakelint on all cmake files in IDF_PATH (except third party)
-#
-# cmakelint: https://github.com/richq/cmake-lint
-#
-# NOTE: This script makes use of features in (currently unreleased)
-# cmakelint >1.4. Install directly from github as follows:
-#
-# pip install https://github.com/richq/cmake-lint/archive/058c6c0ed2536.zip
-#
-
-if [ -z "${IDF_PATH}" ]; then
-    echo "IDF_PATH variable needs to be set"
-    exit 3
-fi
-
-cd "$IDF_PATH"
-
-# Only list the "main" IDF repo, don't check any files in submodules (which may contain
-# third party CMakeLists.txt)
- git ls-tree --full-tree --name-only -r HEAD | grep -v "/third_party/" | grep "^CMakeLists.txt$\|\.cmake$" \
-    | xargs cmakelint --linelength=120 --spaces=4
-
-