From bcc8f2628c593b4ca596ab7d21108ffca00342cc Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 21 Oct 2020 19:30:49 +0800 Subject: [PATCH] CI: Add pre-commit for esp-idf project. add tools/ci/python_packages/idf_ci.py for some util functions used in ci and needs multi-os solution --- .gitlab/CODEOWNERS | 1 + .pre-commit-config.yaml | 60 +++++++++++++ CONTRIBUTING.rst | 2 + tools/ci/check-executable.sh | 66 -------------- tools/ci/check-line-endings.sh | 14 --- .../{codeowners.py => ci/check_codeowners.py} | 8 +- tools/ci/check_deprecated_kconfigs.py | 24 +++--- tools/ci/check_executables.py | 72 ++++++++++++++++ tools/{ => ci}/check_kconfigs.py | 25 ++++-- tools/ci/ci_get_latest_mr_iid.py | 31 ------- tools/ci/ci_get_mr_info.py | 85 +++++++++++++++++++ tools/ci/config/build.yml | 8 +- tools/ci/config/pre_check.yml | 43 +++++----- tools/ci/executable-list.txt | 10 +-- tools/ci/idf_ci_utils.py | 43 ++++++++++ tools/{ => ci}/test_check_kconfigs.py | 0 tools/cmake/run_cmake_lint.sh | 25 ------ 17 files changed, 326 insertions(+), 191 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100755 tools/ci/check-executable.sh delete mode 100755 tools/ci/check-line-endings.sh rename tools/{codeowners.py => ci/check_codeowners.py} (95%) create mode 100755 tools/ci/check_executables.py rename tools/{ => ci}/check_kconfigs.py (97%) delete mode 100644 tools/ci/ci_get_latest_mr_iid.py create mode 100644 tools/ci/ci_get_mr_info.py create mode 100644 tools/ci/idf_ci_utils.py rename tools/{ => ci}/test_check_kconfigs.py (100%) delete mode 100755 tools/cmake/run_cmake_lint.sh diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 0b9baedb14..c591cae4cb 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -48,6 +48,7 @@ /.* @esp-idf-codeowners/tools /.gitlab-ci.yml @esp-idf-codeowners/ci +/.pre-commit-config.yaml @esp-idf-codeowners/ci /.readthedocs.yml @esp-idf-codeowners/docs /CMakeLists.txt @esp-idf-codeowners/build-config /Kconfig @esp-idf-codeowners/build-config diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..41423c3ad0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,60 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 + hooks: + - id: trailing-whitespace + exclude: '.+\.(md|rst)' + - id: end-of-file-fixer + - id: check-executables-have-shebangs + - id: file-contents-sorter + files: 'tools/ci/executable-list.txt' + - id: mixed-line-ending + args: ['-f=lf'] + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: + - id: flake8 + args: ['--config=.flake8', '--tee', '--benchmark'] + - repo: local + hooks: + - id: check-executables + name: Check File Permissions + entry: tools/ci/check_executables.py --action executables + language: python + types: [executable] + exclude: '\.pre-commit/.+' + - id: check-executable-list + name: Validate executable-list.txt + entry: tools/ci/check_executables.py --action list + language: python + pass_filenames: false + always_run: true + - id: check-kconfigs + name: Validate Kconfig files + entry: tools/ci/check_kconfigs.py --exclude-submodules + language: python + pass_filenames: false + always_run: true + - id: check-deprecated-kconfigs-options + name: Check if any Kconfig Options Deprecated + entry: tools/ci/check_deprecated_kconfigs.py --exclude-submodules + language: python + pass_filenames: false + always_run: true + - id: cmake-lint + name: Check CMake Files Format + entry: cmakelint --linelength=120 --spaces=4 + language: python + additional_dependencies: + - https://github.com/richq/cmake-lint/archive/058c6c0ed2536.zip + files: 'CMakeLists.txt$|\.cmake$' + exclude: '\/third_party\/' + - id: check-codeowners + name: Validate Codeowner File + entry: tools/ci/check_codeowners.py ci-check + language: python + files: '\.gitlab/CODEOWNERS' + pass_filenames: false diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e2112a00a7..580f79b676 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -17,6 +17,8 @@ Before sending us a Pull Request, please consider this list of points: * Does any new code conform to the esp-idf :doc:`Style Guide `? +* Have you installed the pre-commit hook for esp-idf? (please refer to https://pre-commit.com/#installation) + * Does the code documentation follow requirements in :doc:`documenting-code`? * Is the code adequately commented for people to understand how it is structured? diff --git a/tools/ci/check-executable.sh b/tools/ci/check-executable.sh deleted file mode 100755 index b40f5fa21d..0000000000 --- a/tools/ci/check-executable.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# This script finds executable files in the repository, excluding some directories, -# then prints the list of all files which are not in executable-list.txt. -# Returns with error if this list is non-empty. -# Also checks if executable-list.txt is sorted and has no duplicates. - -set -o errexit # Exit if command failed. -set -o pipefail # Exit if pipe failed. -set -o nounset # Exit if variable not set. - - -cd $IDF_PATH - -in_list=tools/ci/executable-list.txt -tmp_list=$(mktemp) -out_list=$(mktemp) - -# build exclude pattern like '-o -path ./components/component/submodule' for each submodule -submodule_excludes=$(git config --file .gitmodules --get-regexp path | awk '{ print "-o -path ./" $2 }') - -# figure out which flag to use when searching for executable files -if [ "$(uname -s)" == "Darwin" ]; then - perm_flag="-perm +111" -else - perm_flag="-executable" -fi - -find . -type d \( \ - -path ./.git \ - -o -name build \ - -o -name builds \ - $submodule_excludes \ - \) -prune -o -type f $perm_flag -print \ - | sed "s|^\./||" > $tmp_list - -# this looks for lines present in tmp_list but not in executable-list.txt -comm -13 <(cat $in_list | sed -n "/^#/!p" | sort) <(sort $tmp_list) > $out_list - -ret=0 -if [ -s $out_list ]; then - ret=1 - echo "Error: the following file(s) have executable flag set:" - echo "" - cat $out_list - echo "" - echo "If any files need to be executable (usually, scripts), add them to tools/ci/executable-list.txt" - echo "Make the rest of the files non-executable using 'chmod -x '." - echo "On Windows, use 'git update-index --chmod=-x filename' instead." - echo "" -fi - -if ! diff <(cat $in_list | sed -n "/^#/!p" | sort | uniq) $in_list; then - echo "$in_list is not sorted or has duplicate entries" - ret=2 -fi - -for filename in $(cat $in_list | sed -n "/^#/!p"); do - if [ ! -f "$filename" ]; then - echo "Warning: file '$filename' is present in '$in_list', but does not exist" - fi -done - -rm $tmp_list -rm $out_list - -exit $ret diff --git a/tools/ci/check-line-endings.sh b/tools/ci/check-line-endings.sh deleted file mode 100755 index b813aa6b34..0000000000 --- a/tools/ci/check-line-endings.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -if ! [ -z "$1" ]; then - cd "$1" -fi - -echo "Checking for Windows line endings in `pwd`" - -if git ls-tree --name-only -r HEAD | xargs file -N | grep CRLF; then - echo "Some files have CRLF (Windows-style) line endings. Please convert to LF (Unix-style). git can be configured to do this automatically." - exit 1 -fi - -exit 0 diff --git a/tools/codeowners.py b/tools/ci/check_codeowners.py similarity index 95% rename from tools/codeowners.py rename to tools/ci/check_codeowners.py index 34e9365b45..665f97e47c 100755 --- a/tools/codeowners.py +++ b/tools/ci/check_codeowners.py @@ -22,8 +22,9 @@ import re import subprocess import sys +from idf_ci_utils import IDF_PATH -CODEOWNERS_PATH = os.path.join(os.path.dirname(__file__), "..", ".gitlab", "CODEOWNERS") +CODEOWNERS_PATH = os.path.join(IDF_PATH, ".gitlab", "CODEOWNERS") CODEOWNER_GROUP_PREFIX = "@esp-idf-codeowners/" @@ -31,9 +32,8 @@ def get_all_files(): """ Get list of all file paths in the repository. """ - idf_root = os.path.join(os.path.dirname(__file__), "..") # only split on newlines, since file names may contain spaces - return subprocess.check_output(["git", "ls-files"], cwd=idf_root).decode("utf-8").strip().split('\n') + return subprocess.check_output(["git", "ls-files"], cwd=IDF_PATH).decode("utf-8").strip().split('\n') def pattern_to_regex(pattern): @@ -121,7 +121,7 @@ def action_ci_check(args): errors = [] def add_error(msg): - errors.append("Error at CODEOWNERS:{}: {}".format(line_no, msg)) + errors.append("{}:{}: {}".format(CODEOWNERS_PATH, line_no, msg)) all_files = get_all_files() prev_path_pattern = "" diff --git a/tools/ci/check_deprecated_kconfigs.py b/tools/ci/check_deprecated_kconfigs.py index 8fd4ba2996..b8e55a6e88 100755 --- a/tools/ci/check_deprecated_kconfigs.py +++ b/tools/ci/check_deprecated_kconfigs.py @@ -20,6 +20,7 @@ import argparse import os import sys from io import open +from idf_ci_utils import get_submodule_dirs # FILES_TO_CHECK used as "startswith" pattern to match sdkconfig.defaults variants FILES_TO_CHECK = ('sdkconfig.ci', 'sdkconfig.defaults') @@ -52,10 +53,11 @@ def main(): parser = argparse.ArgumentParser(description='Kconfig options checker') parser.add_argument('--directory', '-d', help='Path to directory to check recursively ' - '(for example $IDF_PATH)', + '(for example $IDF_PATH)', type=_valid_directory, required=default_path is None, default=default_path) + parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules') args = parser.parse_args() # IGNORE_DIRS makes sense when the required directory is IDF_PATH @@ -64,7 +66,12 @@ def main(): ignores = 0 files_to_check = [] deprecated_options = set() - errors = [] + ret = 0 + + ignore_dirs = IGNORE_DIRS + if args.exclude_submodules: + for submodule in get_submodule_dirs(): + ignore_dirs = ignore_dirs + tuple(submodule) for root, dirnames, filenames in os.walk(args.directory): for filename in filenames: @@ -72,7 +79,7 @@ def main(): path_in_idf = os.path.relpath(full_path, args.directory) if filename.startswith(FILES_TO_CHECK): - if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS): + if check_ignore_dirs and path_in_idf.startswith(ignore_dirs): print('{}: Ignored'.format(path_in_idf)) ignores += 1 continue @@ -84,16 +91,13 @@ def main(): used_options = _parse_path(path, '=') used_deprecated_options = deprecated_options & used_options if len(used_deprecated_options) > 0: - errors.append('{}: The following options are deprecated: {}'.format(path, - ', '.join(used_deprecated_options))) + print('{}: The following options are deprecated: {}'.format(path, ', '.join(used_deprecated_options))) + ret = 1 if ignores > 0: print('{} files have been ignored.'.format(ignores)) - - if len(errors) > 0: - print('\n\n'.join(errors)) - sys.exit(1) + return ret if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/tools/ci/check_executables.py b/tools/ci/check_executables.py new file mode 100755 index 0000000000..0b0f1435c8 --- /dev/null +++ b/tools/ci/check_executables.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from sys import exit + + +def _strip_each_item(iterable): + res = [] + for item in iterable: + if item: + res.append(item.strip()) + return res + + +IDF_PATH = os.getenv('IDF_PATH', os.getcwd()) +EXECUTABLE_LIST_FN = os.path.join(IDF_PATH, 'tools/ci/executable-list.txt') +known_executables = _strip_each_item(open(EXECUTABLE_LIST_FN).readlines()) + + +def check_executable_list(): + ret = 0 + for index, fn in enumerate(known_executables): + if not os.path.exists(os.path.join(IDF_PATH, fn)): + print('{}:{} {} not exists. Please remove it manually'.format(EXECUTABLE_LIST_FN, index + 1, fn)) + ret = 1 + return ret + + +def check_executables(files): + ret = 0 + for fn in files: + if fn not in known_executables: + print('"{}" is not in {}'.format(fn, EXECUTABLE_LIST_FN)) + ret = 1 + return ret + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--action', choices=['executables', 'list'], required=True, + help='if "executables", pass all your executables to see if it\'s in the list.' + 'if "list", check if all items on your list exist') + parser.add_argument('filenames', nargs='*', help='Filenames to check.') + args = parser.parse_args() + + if args.action == 'executables': + ret = check_executables(args.filenames) + elif args.action == 'list': + ret = check_executable_list() + else: + raise ValueError + + return ret + + +if __name__ == '__main__': + exit(main()) diff --git a/tools/check_kconfigs.py b/tools/ci/check_kconfigs.py similarity index 97% rename from tools/check_kconfigs.py rename to tools/ci/check_kconfigs.py index 99b65ef4df..0f2b504ed9 100755 --- a/tools/check_kconfigs.py +++ b/tools/ci/check_kconfigs.py @@ -21,6 +21,7 @@ import sys import re import argparse from io import open +from idf_ci_utils import get_submodule_dirs # regular expression for matching Kconfig files RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$' @@ -377,16 +378,22 @@ def main(): parser.add_argument('--verbose', '-v', help='Print more information (useful for debugging)', action='store_true', default=False) parser.add_argument('--directory', '-d', help='Path to directory where Kconfigs should be recursively checked ' - '(for example $IDF_PATH)', + '(for example $IDF_PATH)', type=valid_directory, required=default_path is None, default=default_path) + parser.add_argument('--exclude-submodules', action='store_true', help='Exclude submodules') args = parser.parse_args() - success_couter = 0 + success_counter = 0 ignore_counter = 0 failure = False + ignore_dirs = IGNORE_DIRS + if args.exclude_submodules: + for submodule in get_submodule_dirs(): + ignore_dirs = ignore_dirs + tuple(submodule) + # IGNORE_DIRS makes sense when the required directory is IDF_PATH check_ignore_dirs = default_path is not None and os.path.abspath(args.directory) == os.path.abspath(default_path) @@ -395,7 +402,7 @@ def main(): full_path = os.path.join(root, filename) path_in_idf = os.path.relpath(full_path, args.directory) if re.search(RE_KCONFIG, filename): - if check_ignore_dirs and path_in_idf.startswith(IGNORE_DIRS): + if check_ignore_dirs and path_in_idf.startswith(ignore_dirs): print('{}: Ignored'.format(path_in_idf)) ignore_counter += 1 continue @@ -425,9 +432,9 @@ def main(): 'for solving all issues'.format(path_in_idf + OUTPUT_SUFFIX)) print('Please fix the errors and run {} for checking the correctness of ' 'Kconfigs.'.format(os.path.relpath(os.path.abspath(__file__), args.directory))) - sys.exit(1) + return 1 else: - success_couter += 1 + success_counter += 1 print('{}: OK'.format(path_in_idf)) try: os.remove(suggestions_full_path) @@ -440,10 +447,10 @@ def main(): if ignore_counter > 0: print('{} files have been ignored.'.format(ignore_counter)) - - if success_couter > 0: - print('{} files have been successfully checked.'.format(success_couter)) + if success_counter > 0: + print('{} files have been successfully checked.'.format(success_counter)) + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/tools/ci/ci_get_latest_mr_iid.py b/tools/ci/ci_get_latest_mr_iid.py deleted file mode 100644 index 0b3ed92b88..0000000000 --- a/tools/ci/ci_get_latest_mr_iid.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -# internal use only for CI -# get latest MR IID by source branch - -import argparse -import os - -from gitlab_api import Gitlab - - -def get_MR_IID_by_source_branch(source_branch): - if not source_branch: - return '' - gl = Gitlab(os.getenv('CI_PROJECT_ID')) - if not gl.project: - return '' - mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) - if mrs: - mr = mrs[0] # one source branch can only have one opened MR at one moment - return mr.iid - return '' - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get the latest MR IID by source branch, if not found, return empty string') - parser.add_argument('source_branch', nargs='?', help='source_branch') # won't fail if it's empty - - args = parser.parse_args() - - print(get_MR_IID_by_source_branch(args.source_branch)) diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py new file mode 100644 index 0000000000..b1550e9dbf --- /dev/null +++ b/tools/ci/ci_get_mr_info.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# +# internal use only for CI +# get latest MR information by source branch +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import subprocess + +from gitlab_api import Gitlab + + +def _get_mr_obj(source_branch): + if not source_branch: + return None + gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) + if not gl.project: + return None + mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) + if mrs: + return mrs[0] # one source branch can only have one opened MR at one moment + else: + return None + + +def get_mr_iid(source_branch): # type: (str) -> str + mr = _get_mr_obj(source_branch) + if not mr: + return '' + else: + return str(mr.iid) + + +def get_mr_changed_files(source_branch): + mr = _get_mr_obj(source_branch) + if not mr: + return '' + + return subprocess.check_output(['git', 'diff', '--name-only', + 'origin/{}...origin/{}'.format(mr.target_branch, source_branch)]).decode('utf8') + + +def get_mr_commits(source_branch): + mr = _get_mr_obj(source_branch) + if not mr: + return '' + return '\n'.join([commit.id for commit in mr.commits()]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline') + actions = parser.add_subparsers(dest='action', help='info type') + + common_args = argparse.ArgumentParser(add_help=False) + common_args.add_argument('src_branch', nargs='?', help='source branch') + + actions.add_parser('id', parents=[common_args]) + actions.add_parser('files', parents=[common_args]) + actions.add_parser('commits', parents=[common_args]) + + args = parser.parse_args() + + if args.action == 'id': + print(get_mr_iid(args.src_branch)) + elif args.action == 'files': + print(get_mr_changed_files(args.src_branch)) + elif args.action == 'commits': + print(get_mr_commits(args.src_branch)) + else: + raise NotImplementedError('not possible to get here') diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 0fb66a90ab..a74d1904e5 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -432,9 +432,11 @@ code_quality_check: - .rules:trigger allow_failure: true script: - - export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs` + - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH}) + - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',') # test if this branch have merge request, if not, exit 0 - test -n "$CI_MERGE_REQUEST_IID" || exit 0 + - test -n "$CI_MR_COMMITS" || exit 0 - sonar-scanner -Dsonar.analysis.mode=preview -Dsonar.host.url=$SONAR_HOST_URL @@ -445,12 +447,12 @@ code_quality_check: -Dsonar.projectBaseDir=$CI_PROJECT_DIR -Dsonar.exclusions=$EXCLUSIONS -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.commit_sha=$(git log --pretty=format:%H origin/master..origin/$CI_COMMIT_REF_NAME | tr '\n' ',') + -Dsonar.gitlab.commit_sha=CI_MR_COMMITS -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt -Dsonar.cxx.includeDirectories=components,/usr/include -Dsonar.python.pylint_config=.pylintrc - -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID + -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID -Dsonar.gitlab.merge_request_discussion=true -Dsonar.branch.name=$CI_COMMIT_REF_NAME diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 793917c5a9..f6e68090d2 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -15,15 +15,27 @@ - .pre_check_base_template - .before_script_lesser -check_line_endings: +.check_pre_commit_template: extends: .pre_check_job_template - script: - - tools/ci/check-line-endings.sh ${IDF_PATH} + stage: pre_check + image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1" + before_script: + - source tools/ci/utils.sh + - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -check_permissions: - extends: .pre_check_job_template +check_pre_commit_master_release: + extends: + - .check_pre_commit_template + - .rules:protected script: - - tools/ci/check-executable.sh + - git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA | xargs pre-commit run --files + +check_pre_commit_MR: + extends: + - .check_pre_commit_template + - .rules:dev + script: + - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_COMMIT_BRANCH} | xargs pre-commit run --files check_docs_lang_sync: extends: .pre_check_job_template @@ -79,18 +91,8 @@ check_kconfigs: - tools/*/*/*/Kconfig*.new expire_in: 1 week script: - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py - - ${IDF_PATH}/tools/check_kconfigs.py - -check_deprecated_kconfig_options: - extends: .pre_check_job_template_with_filter - script: - - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py - -check_cmake_style: - extends: .pre_check_job_template - script: - tools/cmake/run_cmake_lint.sh + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py + - ${IDF_PATH}/tools/ci/check_kconfigs.py check_wifi_lib_md5: extends: .pre_check_base_template @@ -183,11 +185,6 @@ clang_tidy_check_all: BOT_NEEDS_TRIGGER_BY_NAME: 1 BOT_LABEL_STATIC_ANALYSIS_ALL: 1 -check_codeowners: - extends: .pre_check_job_template - script: - - tools/codeowners.py ci-check - # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update # the version returned by 'git describe' check_version_tag: diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index be0a9b49ff..97a9915237 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -32,18 +32,18 @@ examples/system/ota/otatool/otatool_example.sh install.fish install.sh tools/build_apps.py -tools/check_kconfigs.py tools/check_python_dependencies.py tools/ci/apply_bot_filter.py tools/ci/build_template_app.sh -tools/ci/check-executable.sh -tools/ci/check-line-endings.sh tools/ci/check_build_warnings.py tools/ci/check_callgraph.py +tools/ci/check_codeowners.py tools/ci/check_deprecated_kconfigs.py tools/ci/check_examples_cmake_make.py tools/ci/check_examples_rom_header.sh +tools/ci/check_executables.py tools/ci/check_idf_version.sh +tools/ci/check_kconfigs.py tools/ci/check_readme_links.py tools/ci/check_rom_apis.sh tools/ci/check_ut_cmake_make.sh @@ -60,11 +60,10 @@ tools/ci/normalize_clangtidy_path.py tools/ci/push_to_github.sh tools/ci/test_build_system.sh tools/ci/test_build_system_cmake.sh +tools/ci/test_check_kconfigs.py tools/ci/test_configure_ci_environment.sh tools/ci/utils.sh tools/cmake/convert_to_cmake.py -tools/cmake/run_cmake_lint.sh -tools/codeowners.py tools/docker/entrypoint.sh tools/docker/hooks/build tools/esp_app_trace/logtrace_proc.py @@ -93,7 +92,6 @@ tools/ldgen/test/test_generation.py tools/mass_mfg/mfg_gen.py tools/mkdfu.py tools/set-submodules-to-github.sh -tools/test_check_kconfigs.py tools/test_idf_monitor/run_test_idf_monitor.py tools/test_idf_py/test_idf_py.py tools/test_idf_size/test.sh diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py new file mode 100644 index 0000000000..e76a9ce23a --- /dev/null +++ b/tools/ci/idf_ci_utils.py @@ -0,0 +1,43 @@ +# internal use only for CI +# some CI related util functions +# +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +import subprocess + +IDF_PATH = os.getenv('IDF_PATH', os.getcwd()) + + +def get_submodule_dirs(): # type: () -> list + """ + To avoid issue could be introduced by multi-os or additional dependency, + we use python and git to get this output + :return: List of submodule dirs + """ + dirs = [] + try: + lines = subprocess.check_output( + ['git', 'config', '--file', os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')), + '--get-regexp', 'path']).decode('utf8').strip().split('\n') + for line in lines: + _, path = line.split(' ') + dirs.append(path) + except Exception as e: + logging.warning(str(e)) + + return dirs diff --git a/tools/test_check_kconfigs.py b/tools/ci/test_check_kconfigs.py similarity index 100% rename from tools/test_check_kconfigs.py rename to tools/ci/test_check_kconfigs.py diff --git a/tools/cmake/run_cmake_lint.sh b/tools/cmake/run_cmake_lint.sh deleted file mode 100755 index 3ba8b9c382..0000000000 --- a/tools/cmake/run_cmake_lint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Run cmakelint on all cmake files in IDF_PATH (except third party) -# -# cmakelint: https://github.com/richq/cmake-lint -# -# NOTE: This script makes use of features in (currently unreleased) -# cmakelint >1.4. Install directly from github as follows: -# -# pip install https://github.com/richq/cmake-lint/archive/058c6c0ed2536.zip -# - -if [ -z "${IDF_PATH}" ]; then - echo "IDF_PATH variable needs to be set" - exit 3 -fi - -cd "$IDF_PATH" - -# Only list the "main" IDF repo, don't check any files in submodules (which may contain -# third party CMakeLists.txt) - git ls-tree --full-tree --name-only -r HEAD | grep -v "/third_party/" | grep "^CMakeLists.txt$\|\.cmake$" \ - | xargs cmakelint --linelength=120 --spaces=4 - -