From e669c7ab5b6c04370763bdca29f8e639823f988f Mon Sep 17 00:00:00 2001 From: "igor.udot" Date: Tue, 20 Feb 2024 11:49:38 +0800 Subject: [PATCH] ci: fast_pipeline yml branch --- .gitlab/ci/build.yml | 1 + .gitlab/ci/common.yml | 2 ++ .gitlab/ci/pre_check.yml | 2 ++ .../Minimal_Template.md | 14 ++++++++ tools/ci/ci_process_description.py | 28 +++++++++++++++ .../scripts/generate_build_child_pipeline.py | 3 ++ .../generate_target_test_child_pipeline.py | 4 +++ .../templates/fast_pipeline.yml | 16 +++++++++ .../templates/generate_target_test_report.yml | 1 + .../templates/test_child_pipeline.yml | 2 ++ tools/ci/exclude_check_tools_files.txt | 2 ++ tools/ci/previous_stage_job_status.py | 35 +++++++++++++++++++ 12 files changed, 110 insertions(+) create mode 100644 tools/ci/ci_process_description.py create mode 100644 tools/ci/dynamic_pipelines/templates/fast_pipeline.yml create mode 100644 tools/ci/previous_stage_job_status.py diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index ae74e4165f..dbc8e91908 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -323,6 +323,7 @@ build_child_pipeline: MR_MODIFIED_FILES: $MR_MODIFIED_FILES PARENT_PIPELINE_ID: $CI_PIPELINE_ID BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS + REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 61137bccfd..b43e1d3d57 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -102,6 +102,8 @@ variables: CCACHE_DIR: "/cache/idf_ccache" CCACHE_MAXSIZE: "50G" + FF_USE_NEW_BASH_EVAL_STRATEGY: "true" + ################################################ # `before_script` and `after_script` Templates # ################################################ diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index a75188c070..09a5ffb3ad 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -135,6 +135,7 @@ pipeline_variables: # MODIFIED_FILES is a list of files that changed, could be used everywhere - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env + - echo "REPORT_EXIT_CODE=0" >> pipeline.env # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only # for non MR pipeline, these are empty lists - | @@ -157,6 +158,7 @@ pipeline_variables: if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env fi + - python tools/ci/ci_process_description.py - cat pipeline.env - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report artifacts: diff --git a/.gitlab/merge_request_templates/Minimal_Template.md b/.gitlab/merge_request_templates/Minimal_Template.md index b020d3cba5..966caf05e7 100644 --- a/.gitlab/merge_request_templates/Minimal_Template.md +++ b/.gitlab/merge_request_templates/Minimal_Template.md @@ -13,3 +13,17 @@ + + diff --git a/tools/ci/ci_process_description.py b/tools/ci/ci_process_description.py new file mode 100644 index 0000000000..4563e42a09 --- /dev/null +++ b/tools/ci/ci_process_description.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import re +import sys + +import yaml + + +if __name__ == '__main__': + description = os.getenv('CI_MERGE_REQUEST_DESCRIPTION', '') + if not description: + sys.exit(0) + pattern = r'^## Dynamic Pipeline Configuration(?:[^`]*?)```(?:\w+)(.*?)```' + result = re.search(pattern, description, re.DOTALL | re.MULTILINE) + if not result: + sys.exit(0) + + data = yaml.safe_load(result.group(1)) + raise_report_exit_code = False + with open('pipeline.env', 'a+') as f: + if 'Test Case Filters' in data: + raise_report_exit_code = True + test_case_filters = ' or '.join(data.get('Test Case Filters')) + f.write(f'TEST_CASE_FILTERS={test_case_filters}\n') + + if raise_report_exit_code: + f.write('REPORT_EXIT_CODE=30\n') diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py index 9994df83c8..62fc2366ff 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -213,6 +213,9 @@ if __name__ == '__main__': args = parser.parse_args() + if test_case_filters := os.getenv('TEST_CASE_FILTERS', None): + args.filter_expr = test_case_filters + if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': print('Build and run all test cases, and compile all cmake apps') args.modified_components = None diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index ef1e28f930..8ec797b615 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -110,6 +110,10 @@ def get_target_test_jobs( else: extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml'] + fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30 + if fast_pipeline_flag: + extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml'] + issues['no_env_marker_test_cases'] = sorted(issues['no_env_marker_test_cases']) issues['no_runner_tags'] = sorted(issues['no_runner_tags']) diff --git a/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml b/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml new file mode 100644 index 0000000000..ea23b84abb --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml @@ -0,0 +1,16 @@ +.generate_pytest_report_base: + stage: .post + tags: [build, shiny] + image: $ESP_ENV_IMAGE + artifacts: + paths: + - target_test_report.html + expire_in: 1 week + when: always + +fast_pipeline:pipeline_ended:always_failed: + when: on_success + extends: .generate_pytest_report_base + script: + - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test + - exit 30 diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml index 88c27ff160..fe54363578 100644 --- a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml +++ b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml @@ -12,3 +12,4 @@ generate_pytest_report: script: - python tools/ci/get_known_failure_cases_file.py - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test + - python tools/ci/previous_stage_job_status.py --stage target_test diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 336b74f0a6..0378bdd381 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -11,6 +11,7 @@ generate_pytest_build_report: - test_related_apps_download_urls.yml script: - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build + - python tools/ci/previous_stage_job_status.py --stage build generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage @@ -31,6 +32,7 @@ Pytest Target Test Jobs: - generate_pytest_child_pipeline variables: PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID + REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 4839470759..c5d24295e0 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -13,6 +13,7 @@ tools/ci/astyle-rules.yml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py tools/ci/ci_get_mr_info.py +tools/ci/ci_process_description.py tools/ci/configure_ci_environment.sh tools/ci/generate_rules.py tools/ci/deploy_docs.py @@ -50,3 +51,4 @@ tools/ci/python_packages/idf_iperf_test_util/**/* tools/esp_prov/**/* tools/ci/sort_yaml.py tools/ci/sg_rules/* +tools/ci/previous_stage_job_status.py diff --git a/tools/ci/previous_stage_job_status.py b/tools/ci/previous_stage_job_status.py new file mode 100644 index 0000000000..505cf1ab5c --- /dev/null +++ b/tools/ci/previous_stage_job_status.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import sys + +import requests + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--stage', type=str, help='Stage name for check jobs status') + args = parser.parse_args() + + GITLAB_TOKEN = os.getenv('ESPCI_TOKEN') + GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER') + CI_PROJECT_ID = os.getenv('CI_PROJECT_ID') + CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID') + + api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100' + + page = 0 + while True: + response = requests.get( + f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}', + headers={'PRIVATE-TOKEN': GITLAB_TOKEN} + ) + jobs = response.json() + if not jobs: + break + + for job in jobs: + if job['stage'] == args.stage: + sys.exit(1) + page += 1