Merge branch 'ci/test_correct_exit_code' into 'master'

Add support for Dynamic Pipeline Configuration

Closes IDFCI-2035 and IDFCI-1936

See merge request espressif/esp-idf!29115

.gitlab/ci/build.yml

@@ -323,6 +323,7 @@ build_child_pipeline:
     MR_MODIFIED_FILES: $MR_MODIFIED_FILES
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
     BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
+    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

.gitlab/ci/common.yml

@@ -102,6 +102,8 @@ variables:
   CCACHE_DIR: "/cache/idf_ccache"
   CCACHE_MAXSIZE: "50G"
 
+  FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
+
 ################################################
 # `before_script` and `after_script` Templates #
 ################################################

.gitlab/ci/pre_check.yml

@@ -135,6 +135,7 @@ pipeline_variables:
     # MODIFIED_FILES is a list of files that changed, could be used everywhere
     - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
     - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
+    - echo "REPORT_EXIT_CODE=0" >> pipeline.env
     # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
     # for non MR pipeline, these are empty lists
     - |
@@ -157,6 +158,7 @@ pipeline_variables:
       if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
         echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
       fi
+    - python tools/ci/ci_process_description.py
     - cat pipeline.env
     - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
   artifacts:

.gitlab/merge_request_templates/Minimal_Template.md

@@ -13,3 +13,17 @@
 <!-- Either state release notes or write "No release notes" -->
 
 <!-- ## Breaking change notes --><!-- Optional -->
+
+<!-- ## Dynamic Pipeline Configuration
+```yaml
+Test Case Filters:
+    # Only run tests that match the given substring expression (modified files/components will be ignored):
+    # Please use a list of strings. 
+    # This will run the test cases filtered like `pytest -k "(<list_item_1>) or (<list_item_2>) or ...`
+    # The fast pipeline will fail at the final stage.
+    # For example:
+      - test_sdm and not sdmmc
+      - test_hello_world
+    # This example will include all tests containing 'test_hello_world' in the name,
+    # and include all tests containing 'test_sdm' but not 'sdmmc' in the name.
+``` --><!-- Optional -->

tools/ci/ci_process_description.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+import re
+import sys
+
+import yaml
+
+
+if __name__ == '__main__':
+    description = os.getenv('CI_MERGE_REQUEST_DESCRIPTION', '')
+    if not description:
+        sys.exit(0)
+    pattern = r'^## Dynamic Pipeline Configuration(?:[^`]*?)```(?:\w+)(.*?)```'
+    result = re.search(pattern, description, re.DOTALL | re.MULTILINE)
+    if not result:
+        sys.exit(0)
+
+    data = yaml.safe_load(result.group(1))
+    raise_report_exit_code = False
+    with open('pipeline.env', 'a+') as f:
+        if 'Test Case Filters' in data:
+            raise_report_exit_code = True
+            test_case_filters = ' or '.join(data.get('Test Case Filters'))
+            f.write(f'TEST_CASE_FILTERS={test_case_filters}\n')
+
+        if raise_report_exit_code:
+            f.write('REPORT_EXIT_CODE=30\n')

tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py

@@ -213,6 +213,9 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
+    if test_case_filters := os.getenv('TEST_CASE_FILTERS', None):
+        args.filter_expr = test_case_filters
+
     if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1':
         print('Build and run all test cases, and compile all cmake apps')
         args.modified_components = None

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -110,6 +110,10 @@ def get_target_test_jobs(
     else:
         extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml']
 
+    fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30
+    if fast_pipeline_flag:
+        extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml']
+
     issues['no_env_marker_test_cases'] = sorted(issues['no_env_marker_test_cases'])
     issues['no_runner_tags'] = sorted(issues['no_runner_tags'])
 

tools/ci/dynamic_pipelines/templates/fast_pipeline.yml

@@ -0,0 +1,16 @@
+.generate_pytest_report_base:
+  stage: .post
+  tags: [build, shiny]
+  image: $ESP_ENV_IMAGE
+  artifacts:
+    paths:
+      - target_test_report.html
+    expire_in: 1 week
+    when: always
+
+fast_pipeline:pipeline_ended:always_failed:
+  when: on_success
+  extends: .generate_pytest_report_base
+  script:
+      - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test
+      - exit 30

tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml

@@ -12,3 +12,4 @@ generate_pytest_report:
   script:
       - python tools/ci/get_known_failure_cases_file.py
       - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test
+      - python tools/ci/previous_stage_job_status.py --stage target_test

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -11,6 +11,7 @@ generate_pytest_build_report:
       - test_related_apps_download_urls.yml
   script:
     - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build
+    - python tools/ci/previous_stage_job_status.py --stage build
 
 generate_pytest_child_pipeline:
   # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage
@@ -31,6 +32,7 @@ Pytest Target Test Jobs:
     - generate_pytest_child_pipeline
   variables:
     PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID
+    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
   # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
   inherit:
     variables: false

tools/ci/exclude_check_tools_files.txt

@@ -13,6 +13,7 @@ tools/ci/astyle-rules.yml
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py
 tools/ci/ci_get_mr_info.py
+tools/ci/ci_process_description.py
 tools/ci/configure_ci_environment.sh
 tools/ci/generate_rules.py
 tools/ci/deploy_docs.py
@@ -50,3 +51,4 @@ tools/ci/python_packages/idf_iperf_test_util/**/*
 tools/esp_prov/**/*
 tools/ci/sort_yaml.py
 tools/ci/sg_rules/*
+tools/ci/previous_stage_job_status.py

tools/ci/previous_stage_job_status.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import argparse
+import os
+import sys
+
+import requests
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--stage', type=str, help='Stage name for check jobs status')
+    args = parser.parse_args()
+
+    GITLAB_TOKEN = os.getenv('ESPCI_TOKEN')
+    GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER')
+    CI_PROJECT_ID = os.getenv('CI_PROJECT_ID')
+    CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID')
+
+    api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100'
+
+    page = 0
+    while True:
+        response = requests.get(
+            f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}',
+            headers={'PRIVATE-TOKEN': GITLAB_TOKEN}
+        )
+        jobs = response.json()
+        if not jobs:
+            break
+
+        for job in jobs:
+            if job['stage'] == args.stage:
+                sys.exit(1)
+        page += 1