Merge branch 'ci/test_correct_exit_code' into 'master'

Add support for Dynamic Pipeline Configuration

Closes IDFCI-2035 and IDFCI-1936

See merge request espressif/esp-idf!29115
This commit is contained in:
Igor Udot 2024-08-07 11:04:07 +08:00
commit b40ac839cc
12 changed files with 110 additions and 0 deletions

View File

@ -323,6 +323,7 @@ build_child_pipeline:
MR_MODIFIED_FILES: $MR_MODIFIED_FILES
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
REPORT_EXIT_CODE: $REPORT_EXIT_CODE
# https://gitlab.com/gitlab-org/gitlab/-/issues/214340
inherit:
variables: false

View File

@ -102,6 +102,8 @@ variables:
CCACHE_DIR: "/cache/idf_ccache"
CCACHE_MAXSIZE: "50G"
FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
################################################
# `before_script` and `after_script` Templates #
################################################

View File

@ -135,6 +135,7 @@ pipeline_variables:
# MODIFIED_FILES is a list of files that changed, could be used everywhere
- MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
- echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
- echo "REPORT_EXIT_CODE=0" >> pipeline.env
# MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
# for non MR pipeline, these are empty lists
- |
@ -157,6 +158,7 @@ pipeline_variables:
if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
fi
- python tools/ci/ci_process_description.py
- cat pipeline.env
- python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
artifacts:

View File

@ -13,3 +13,17 @@
<!-- Either state release notes or write "No release notes" -->
<!-- ## Breaking change notes --><!-- Optional -->
<!-- ## Dynamic Pipeline Configuration
```yaml
Test Case Filters:
# Only run tests that match the given substring expression (modified files/components will be ignored):
# Please use a list of strings.
# This will run the test cases filtered like `pytest -k "(<list_item_1>) or (<list_item_2>) or ...`
# The fast pipeline will fail at the final stage.
# For example:
- test_sdm and not sdmmc
- test_hello_world
# This example will include all tests containing 'test_hello_world' in the name,
# and include all tests containing 'test_sdm' but not 'sdmmc' in the name.
``` --><!-- Optional -->

View File

@ -0,0 +1,28 @@
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
import re
import sys
import yaml
if __name__ == '__main__':
description = os.getenv('CI_MERGE_REQUEST_DESCRIPTION', '')
if not description:
sys.exit(0)
pattern = r'^## Dynamic Pipeline Configuration(?:[^`]*?)```(?:\w+)(.*?)```'
result = re.search(pattern, description, re.DOTALL | re.MULTILINE)
if not result:
sys.exit(0)
data = yaml.safe_load(result.group(1))
raise_report_exit_code = False
with open('pipeline.env', 'a+') as f:
if 'Test Case Filters' in data:
raise_report_exit_code = True
test_case_filters = ' or '.join(data.get('Test Case Filters'))
f.write(f'TEST_CASE_FILTERS={test_case_filters}\n')
if raise_report_exit_code:
f.write('REPORT_EXIT_CODE=30\n')

View File

@ -213,6 +213,9 @@ if __name__ == '__main__':
args = parser.parse_args()
if test_case_filters := os.getenv('TEST_CASE_FILTERS', None):
args.filter_expr = test_case_filters
if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
print('Build and run all test cases, and compile all cmake apps')
args.modified_components = None

View File

@ -110,6 +110,10 @@ def get_target_test_jobs(
else:
extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml']
fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30
if fast_pipeline_flag:
extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml']
issues['no_env_marker_test_cases'] = sorted(issues['no_env_marker_test_cases'])
issues['no_runner_tags'] = sorted(issues['no_runner_tags'])

View File

@ -0,0 +1,16 @@
.generate_pytest_report_base:
stage: .post
tags: [build, shiny]
image: $ESP_ENV_IMAGE
artifacts:
paths:
- target_test_report.html
expire_in: 1 week
when: always
fast_pipeline:pipeline_ended:always_failed:
when: on_success
extends: .generate_pytest_report_base
script:
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test
- exit 30

View File

@ -12,3 +12,4 @@ generate_pytest_report:
script:
- python tools/ci/get_known_failure_cases_file.py
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test
- python tools/ci/previous_stage_job_status.py --stage target_test

View File

@ -11,6 +11,7 @@ generate_pytest_build_report:
- test_related_apps_download_urls.yml
script:
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build
- python tools/ci/previous_stage_job_status.py --stage build
generate_pytest_child_pipeline:
# finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage
@ -31,6 +32,7 @@ Pytest Target Test Jobs:
- generate_pytest_child_pipeline
variables:
PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID
REPORT_EXIT_CODE: $REPORT_EXIT_CODE
# https://gitlab.com/gitlab-org/gitlab/-/issues/214340
inherit:
variables: false

View File

@ -13,6 +13,7 @@ tools/ci/astyle-rules.yml
tools/ci/checkout_project_ref.py
tools/ci/ci_fetch_submodule.py
tools/ci/ci_get_mr_info.py
tools/ci/ci_process_description.py
tools/ci/configure_ci_environment.sh
tools/ci/generate_rules.py
tools/ci/deploy_docs.py
@ -50,3 +51,4 @@ tools/ci/python_packages/idf_iperf_test_util/**/*
tools/esp_prov/**/*
tools/ci/sort_yaml.py
tools/ci/sg_rules/*
tools/ci/previous_stage_job_status.py

View File

@ -0,0 +1,35 @@
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import os
import sys
import requests
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--stage', type=str, help='Stage name for check jobs status')
args = parser.parse_args()
GITLAB_TOKEN = os.getenv('ESPCI_TOKEN')
GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER')
CI_PROJECT_ID = os.getenv('CI_PROJECT_ID')
CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID')
api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100'
page = 0
while True:
response = requests.get(
f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}',
headers={'PRIVATE-TOKEN': GITLAB_TOKEN}
)
jobs = response.json()
if not jobs:
break
for job in jobs:
if job['stage'] == args.stage:
sys.exit(1)
page += 1