diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml index 0c28b54751..fe1ea90f83 100644 --- a/.gitlab/ci/post_deploy.yml +++ b/.gitlab/ci/post_deploy.yml @@ -3,6 +3,7 @@ generate_failed_jobs_report: tags: [build, shiny] image: $ESP_ENV_IMAGE when: always + dependencies: [] # Do not download artifacts from the previous stages artifacts: expire_in: 1 week when: always diff --git a/conftest.py b/conftest.py index 7d8290b071..af6d356a0e 100644 --- a/conftest.py +++ b/conftest.py @@ -252,6 +252,34 @@ def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: request.node.funcargs['test_case_name'] = test_case_name +@pytest.fixture(autouse=True) +def set_dut_log_url(record_xml_attribute: t.Callable[[str, object], None], _pexpect_logfile: str) -> t.Generator: + # Record the "dut_log_url" attribute in the XML report once test execution finished + yield + + if not isinstance(_pexpect_logfile, str): + record_xml_attribute('dut_log_url', 'No log URL found') + return + + ci_pages_url = os.getenv('CI_PAGES_URL') + logdir_pattern = re.compile(rf'({DEFAULT_LOGDIR}/.*)') + match = logdir_pattern.search(_pexpect_logfile) + + if not match: + record_xml_attribute('dut_log_url', 'No log URL found') + return + + if not ci_pages_url: + record_xml_attribute('dut_log_url', _pexpect_logfile) + return + + job_id = os.getenv('CI_JOB_ID', '0') + modified_ci_pages_url = ci_pages_url.replace('esp-idf', '-/esp-idf') + log_url = f'{modified_ci_pages_url}/-/jobs/{job_id}/artifacts/{match.group(1)}' + + record_xml_attribute('dut_log_url', log_url) + + ###################### # Log Util Functions # ###################### diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index e661e3ded3..b40d827448 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -166,6 +166,7 @@ class TestCase: 'time': float(node.attrib.get('time') or 0), 'ci_job_url': node.attrib.get('ci_job_url') or '', 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', + 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', } failure_node = node.find('failure') diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index dd3951c107..be8fce0af3 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -1,11 +1,13 @@ # SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import abc +import copy import fnmatch import html import os import re import typing as t +from textwrap import dedent import yaml from artifacts_handler import ArtifactType @@ -21,20 +23,24 @@ from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME from .models import GitlabJob from .models import TestCase from .utils import fetch_failed_testcases_failure_ratio +from .utils import format_permalink +from .utils import get_report_url from .utils import is_url from .utils import load_known_failure_cases class ReportGenerator: - REGEX_PATTERN = '#### {}[^####]+' + REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)' - def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, *, title: str): + def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, job_id: int, commit_id: str, *, title: str): gl_project = Gitlab(project_id).project if mr_iid is not None: self.mr = gl_project.mergerequests.get(mr_iid) else: self.mr = None self.pipeline_id = pipeline_id + self.job_id = job_id + self.commit_id = commit_id self.title = title self.output_filepath = self.title.lower().replace(' ', '_') + '.html' @@ -47,10 +53,30 @@ class ReportGenerator: return '' + def write_report_to_file(self, report_str: str, job_id: int, output_filepath: str) -> t.Optional[str]: + """ + Writes the report to a file and constructs a modified URL based on environment settings. + + :param report_str: The report content to be written to the file. + :param job_id: The job identifier used to construct the URL. + :param output_filepath: The path to the output file. + :return: The modified URL pointing to the job's artifacts. + """ + if not report_str: + return None + with open(output_filepath, 'w') as file: + file.write(report_str) + + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` + report_url: str = get_report_url(job_id, output_filepath) + return report_url + def generate_html_report(self, table_str: str) -> str: # we're using bootstrap table - table_str = table_str.replace('', '
') - + table_str = table_str.replace( + '
', '
' + ) with open(REPORT_TEMPLATE_FILEPATH) as fr: template = fr.read() @@ -62,19 +88,16 @@ class ReportGenerator: def create_table_section( self, - report_sections: list, title: str, items: list, headers: list, row_attrs: list, value_functions: t.Optional[list] = None, - ) -> None: + ) -> t.List: """ Appends a formatted section to a report based on the provided items. This section includes a header and a table constructed from the items list with specified headers and attributes. - :param report_sections: List where the HTML report sections are collected. This list is - modified in-place by appending new sections. :param title: Title for the report section. This title is used as a header above the table. :param items: List of item objects to include in the table. Each item should have attributes that correspond to the row_attrs and value_functions specified. @@ -86,17 +109,34 @@ class ReportGenerator: a function that takes an item and returns a string. This is used for generating dynamic columns based on item data. - :return: None. The function modifies the 'report_sections' list by appending new HTML sections. + :return: List with appended HTML sections. """ if not items: - return + return [] - report_sections.append(f'

{title}

') - report_sections.append( + report_sections = [ + f"""

{title}

""", self._create_table_for_items( items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or [] - ) - ) + ), + ] + return report_sections + + @staticmethod + def generate_additional_info_section(title: str, count: int, report_url: t.Optional[str] = None) -> str: + """ + Generate a section for the additional info string. + + :param title: The title of the section. + :param count: The count of test cases. + :param report_url: The URL of the report. If count = 0, only the count will be included. + :return: The formatted additional info section string. + """ + if count != 0 and report_url: + return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' + else: + return f'- **{title}:** {count}\n' def _create_table_for_items( self, @@ -185,7 +225,7 @@ class ReportGenerator: def _get_report_str(self) -> str: raise NotImplementedError - def post_report(self, job_id: int, commit_id: str) -> None: + def post_report(self, print_report_path: bool = True) -> None: # report in html format, otherwise will exceed the limit comment = f'#### {self.title}\n' @@ -194,18 +234,12 @@ class ReportGenerator: if self.additional_info: comment += f'{self.additional_info}\n' - if report_str: - with open(self.output_filepath, 'w') as fw: - fw.write(report_str) + report_url_path = self.write_report_to_file(report_str, self.job_id, self.output_filepath) + if print_report_path and report_url_path: + comment += dedent(f""" + Full {self.title} here: {report_url_path} (with commit {self.commit_id[:8]} - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt - # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') - - comment += f""" -Full {self.title} here: {url}/-/jobs/{job_id}/artifacts/{self.output_filepath} (with commit {commit_id[:8]}) - -""" + """) print(comment) if self.mr is None: @@ -234,11 +268,13 @@ class BuildReportGenerator(ReportGenerator): project_id: int, mr_iid: int, pipeline_id: int, + job_id: int, + commit_id: str, *, title: str = 'Build Report', apps: t.List[App], ): - super().__init__(project_id, mr_iid, pipeline_id, title=title) + super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) self.apps = apps self.apps_presigned_url_filepath = TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME @@ -365,14 +401,26 @@ class TargetTestReportGenerator(ReportGenerator): project_id: int, mr_iid: int, pipeline_id: int, + job_id: int, + commit_id: str, *, title: str = 'Target Test Report', test_cases: t.List[TestCase], ): - super().__init__(project_id, mr_iid, pipeline_id, title=title) + super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) self.test_cases = test_cases self._known_failure_cases_set = None + self.report_titles_map = { + 'failed_yours': 'Failed Test Cases on Your branch (Excludes Known Failure Cases)', + 'failed_others': 'Failed Test Cases on Other branches (Excludes Known Failure Cases)', + 'failed_known': 'Known Failure Cases', + 'skipped': 'Skipped Test Cases', + 'succeeded': 'Succeeded Test Cases', + } + self.skipped_test_cases_report_file = 'skipped_cases.html' + self.succeeded_cases_report_file = 'succeeded_cases.html' + self.failed_cases_report_file = 'failed_cases.html' @property def known_failure_cases_set(self) -> t.Optional[t.Set[str]]: @@ -382,6 +430,10 @@ class TargetTestReportGenerator(ReportGenerator): return self._known_failure_cases_set def get_known_failure_cases(self) -> t.List[TestCase]: + """ + Retrieve the known failure test cases. + :return: A list of known failure test cases. + """ if self.known_failure_cases_set is None: return [] matched_cases = [ @@ -392,109 +444,187 @@ class TargetTestReportGenerator(ReportGenerator): ] return matched_cases + @staticmethod + def filter_test_cases( + cur_branch_failures: t.List[TestCase], + other_branch_failures: t.List[TestCase], + ) -> t.Tuple[t.List[TestCase], t.List[TestCase]]: + """ + Filter the test cases into current branch failures and other branch failures. + + :param cur_branch_failures: List of failed test cases on the current branch. + :param other_branch_failures: List of failed test cases on other branches. + :return: A tuple containing two lists: + - failed_test_cases_cur_branch_only: Test cases that have failed only on the current branch. + - failed_test_cases_other_branch_exclude_cur_branch: Test cases that have failed on other branches + excluding the current branch. + """ + cur_branch_unique_failures = [] + other_branch_failure_map = {tc.name: tc for tc in other_branch_failures} + + for cur_tc in cur_branch_failures: + if cur_tc.latest_failed_count > 0 and ( + cur_tc.name not in other_branch_failure_map + or other_branch_failure_map[cur_tc.name].latest_failed_count == 0 + ): + cur_branch_unique_failures.append(cur_tc) + uniq_fail_names = {cur_tc.name for cur_tc in cur_branch_unique_failures} + other_branch_exclusive_failures = [tc for tc in other_branch_failures if tc.name not in uniq_fail_names] + + return cur_branch_unique_failures, other_branch_exclusive_failures + + def get_failed_cases_report_parts(self) -> t.List[str]: + """ + Generate the report parts for failed test cases and update the additional info section. + :return: A list of strings representing the table sections for the failed test cases. + """ + known_failures = self.get_known_failure_cases() + failed_test_cases = self._filter_items( + self.test_cases, lambda tc: tc.is_failure and tc.name not in {case.name for case in known_failures} + ) + failed_test_cases_cur_branch = self._sort_items( + fetch_failed_testcases_failure_ratio( + copy.deepcopy(failed_test_cases), + branches_filter={'include_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, + ), + key='latest_failed_count', + ) + failed_test_cases_other_branch = self._sort_items( + fetch_failed_testcases_failure_ratio( + copy.deepcopy(failed_test_cases), + branches_filter={'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, + ), + key='latest_failed_count', + ) + failed_test_cases_cur_branch, failed_test_cases_other_branch = self.filter_test_cases( + failed_test_cases_cur_branch, failed_test_cases_other_branch + ) + cur_branch_cases_table_section = self.create_table_section( + title=self.report_titles_map['failed_yours'], + items=failed_test_cases_cur_branch, + headers=[ + 'Test Case', + 'Test Script File Path', + 'Failure Reason', + f'Failures on your branch (40 latest testcases)', + 'Dut Log URL', + 'Job URL', + 'Grafana URL', + ], + row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures on your branch (40 latest testcases)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) + other_branch_cases_table_section = self.create_table_section( + title=self.report_titles_map['failed_others'], + items=failed_test_cases_other_branch, + headers=[ + 'Test Case', + 'Test Script File Path', + 'Failure Reason', + 'Failures across all other branches (40 latest testcases)', + 'Dut Log URL', + 'Job URL', + 'Grafana URL', + ], + row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures across all other branches (40 latest testcases)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) + known_failures_cases_table_section = self.create_table_section( + title=self.report_titles_map['failed_known'], + items=known_failures, + headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], + ) + failed_cases_report_url = self.write_report_to_file( + self.generate_html_report( + ''.join( + cur_branch_cases_table_section + + other_branch_cases_table_section + + known_failures_cases_table_section + ) + ), + self.job_id, + self.failed_cases_report_file, + ) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['failed_yours'], len(failed_test_cases_cur_branch), failed_cases_report_url + ) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['failed_others'], len(failed_test_cases_other_branch), failed_cases_report_url + ) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['failed_known'], len(known_failures), failed_cases_report_url + ) + return cur_branch_cases_table_section + other_branch_cases_table_section + known_failures_cases_table_section + + def get_skipped_cases_report_parts(self) -> t.List[str]: + """ + Generate the report parts for skipped test cases and update the additional info section. + :return: A list of strings representing the table sections for the skipped test cases. + """ + skipped_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_skipped) + skipped_cases_table_section = self.create_table_section( + title=self.report_titles_map['skipped'], + items=skipped_test_cases, + headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], + row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], + ) + skipped_cases_report_url = self.write_report_to_file( + self.generate_html_report(''.join(skipped_cases_table_section)), + self.job_id, + self.skipped_test_cases_report_file, + ) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['skipped'], len(skipped_test_cases), skipped_cases_report_url + ) + return skipped_cases_table_section + + def get_succeeded_cases_report_parts(self) -> t.List[str]: + """ + Generate the report parts for succeeded test cases and update the additional info section. + :return: A list of strings representing the table sections for the succeeded test cases. + """ + succeeded_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_success) + succeeded_cases_table_section = self.create_table_section( + title=self.report_titles_map['succeeded'], + items=succeeded_test_cases, + headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], + ) + succeeded_cases_report_url = self.write_report_to_file( + self.generate_html_report(''.join(succeeded_cases_table_section)), + self.job_id, + self.succeeded_cases_report_file, + ) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['succeeded'], len(succeeded_test_cases), succeeded_cases_report_url + ) + self.additional_info += '\n' + return succeeded_cases_table_section + def _get_report_str(self) -> str: """ Generate a complete HTML report string by processing test cases. :return: Complete HTML report string. """ - report_parts: list = [] + self.additional_info = f'**Test Case Summary (with commit {self.commit_id[:8]}):**\n' + failed_cases_report_parts = self.get_failed_cases_report_parts() + skipped_cases_report_parts = self.get_skipped_cases_report_parts() + succeeded_cases_report_parts = self.get_succeeded_cases_report_parts() - known_failures = self.get_known_failure_cases() - known_failure_case_names = {case.name for case in known_failures} - failed_test_cases = self._filter_items( - self.test_cases, lambda tc: tc.is_failure and tc.name not in known_failure_case_names + return self.generate_html_report( + ''.join(failed_cases_report_parts + skipped_cases_report_parts + succeeded_cases_report_parts) ) - failed_test_cases_with_ratio = self._sort_items( - fetch_failed_testcases_failure_ratio(failed_test_cases), key='latest_failed_count' - ) - skipped_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_skipped) - successful_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_success) - - current_branch_failures = self._sort_items( - self._filter_items(failed_test_cases_with_ratio, lambda tc: tc.latest_failed_count == 0), - key='latest_failed_count', - ) - other_branch_failures = self._sort_items( - self._filter_items( - failed_test_cases_with_ratio, lambda tc: tc.name not in [t.name for t in current_branch_failures] - ), - key='latest_failed_count', - ) - - self.create_table_section( - report_sections=report_parts, - title='Failed Test Cases on Your branch (Excludes Known Failure Cases)', - items=current_branch_failures, - headers=[ - 'Test Case', - 'Test Script File Path', - 'Failure Reason', - 'Failures across all other branches (20 latest testcases)', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (20 latest testcases)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], - ) - self.create_table_section( - report_sections=report_parts, - title='Failed Test Cases on Other branches (Excludes Known Failure Cases)', - items=other_branch_failures, - headers=[ - 'Test Case', - 'Test Script File Path', - 'Failure Reason', - 'Failures across all other branches (20 latest testcases)', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (20 latest testcases)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], - ) - - self.create_table_section( - report_sections=report_parts, - title='Known Failure Cases', - items=known_failures, - headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - ) - self.create_table_section( - report_sections=report_parts, - title='Skipped Test Cases', - items=skipped_test_cases, - headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], - row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], - ) - self.create_table_section( - report_sections=report_parts, - title='Succeeded Test Cases', - items=successful_test_cases, - headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], - ) - - self.additional_info = ( - '**Test Case Summary:**\n' - f'- **Failed Test Cases on Your Branch (Excludes Known Failure Cases):** {len(current_branch_failures)}.\n' - f'- **Failed Test Cases on Other Branches (Excludes Known Failure Cases):** {len(other_branch_failures)}.\n' - f'- **Known Failures:** {len(known_failures)}\n' - f'- **Skipped Test Cases:** {len(skipped_test_cases)}\n' - f'- **Succeeded Test Cases:** {len(successful_test_cases)}\n\n' - 'Please check report below for more information.\n\n' - ) - - return self.generate_html_report(''.join(report_parts)) class JobReportGenerator(ReportGenerator): @@ -503,12 +633,19 @@ class JobReportGenerator(ReportGenerator): project_id: int, mr_iid: int, pipeline_id: int, + job_id: int, + commit_id: str, *, title: str = 'Job Report', jobs: t.List[GitlabJob], ): - super().__init__(project_id, mr_iid, pipeline_id, title=title) + super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) self.jobs = jobs + self.report_titles_map = { + 'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)', + 'succeeded': 'Succeeded Jobs', + } + self.failed_jobs_report_file = 'job_report.html' def _get_report_str(self) -> str: """ @@ -516,7 +653,6 @@ class JobReportGenerator(ReportGenerator): :return: Complete HTML report string. """ report_str: str = '' - report_parts: list = [] if not self.jobs: print('No jobs found, skip generating job report') @@ -530,34 +666,41 @@ class JobReportGenerator(ReportGenerator): ) succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) - self.additional_info = ( - '**Job Summary:**\n' - f'- **Failed Jobs (Excludes "integration_test" and "target_test" jobs):** {len(relevant_failed_jobs)}\n' - f'- **Succeeded Jobs:** {len(succeeded_jobs)}\n\n' + self.additional_info = f'**Job Summary (with commit {self.commit_id[:8]}):**\n' + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['succeeded'], len(succeeded_jobs) ) - if relevant_failed_jobs: - self.create_table_section( - report_sections=report_parts, - title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - items=relevant_failed_jobs, - headers=[ - 'Job Name', - 'Failure Reason', - 'Failure Log', - 'Failures across all other branches (10 latest jobs)', - 'URL', - 'CI Dashboard URL', - ], - row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (10 latest jobs)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], + if not relevant_failed_jobs: + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['failed_jobs'], len(relevant_failed_jobs) ) - self.additional_info += f'Please check report below for more information.\n\n' - report_str = self.generate_html_report(''.join(report_parts)) + return report_str + + report_sections = self.create_table_section( + title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', + items=relevant_failed_jobs, + headers=[ + 'Job Name', + 'Failure Reason', + 'Failure Log', + 'Failures across all other branches (10 latest jobs)', + 'URL', + 'CI Dashboard URL', + ], + row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures across all other branches (10 latest jobs)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) + relevant_failed_jobs_report_url = get_report_url(self.job_id, self.failed_jobs_report_file) + self.additional_info += self.generate_additional_info_section( + self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url + ) + + report_str = self.generate_html_report(''.join(report_sections)) return report_str diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py index bb2996d191..5d2a361b94 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_report.py @@ -74,17 +74,17 @@ def generate_build_report(args: argparse.Namespace) -> None: app for file_name in glob.glob(args.app_list_filepattern) for app in import_apps_from_txt(file_name) ] report_generator = BuildReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, apps=apps + args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, apps=apps ) - report_generator.post_report(args.job_id, args.commit_id) + report_generator.post_report() def generate_target_test_report(args: argparse.Namespace) -> None: test_cases: t.List[t.Any] = parse_testcases_from_filepattern(args.junit_report_filepattern) report_generator = TargetTestReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, test_cases=test_cases + args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, test_cases=test_cases ) - report_generator.post_report(args.job_id, args.commit_id) + report_generator.post_report(print_report_path=False) def generate_jobs_report(args: argparse.Namespace) -> None: @@ -93,8 +93,8 @@ def generate_jobs_report(args: argparse.Namespace) -> None: if not jobs: return - report_generator = JobReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, jobs=jobs) - report_generator.post_report(args.job_id, args.commit_id) + report_generator = JobReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, jobs=jobs) + report_generator.post_report(print_report_path=False) if __name__ == '__main__': diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml index 8fe17af72e..88c27ff160 100644 --- a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml +++ b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml @@ -6,6 +6,9 @@ generate_pytest_report: artifacts: paths: - target_test_report.html + - failed_cases.html + - skipped_cases.html + - succeeded_cases.html script: - python tools/ci/get_known_failure_cases_file.py - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html index 6997fa45c1..bb35367f94 100644 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -5,18 +5,29 @@ {{title}} + + @@ -24,8 +35,29 @@ + + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml index 8f3737b75d..0334a68155 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml @@ -1,7 +1,7 @@ - + conftest.py:74: in case_tester yield CaseTester(dut, **kwargs) tools/ci/idf_unity_tester.py:202: in __init__ @@ -18,7 +18,7 @@ tools/ci/idf_unity_tester.py:202: in __init__ raise EOFError E EOFError - + conftest.py:74: in case_tester yield CaseTester(dut, **kwargs) tools/ci/idf_unity_tester.py:202: in __init__ @@ -37,9 +37,9 @@ E EOFError - - - + + + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1272: in pytest_runtest_call self._raise_dut_failed_cases_if_exists(duts) # type: ignore /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1207: in _raise_dut_failed_cases_if_exists @@ -48,19 +48,19 @@ E AssertionError: Unity test failed - - + + /builds/espressif/esp-idf/tools/test_build_system/test_common.py:134: Linux does not support executing .exe files - - - - - - + + + + + + - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -95,7 +95,7 @@ E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of t E Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -128,7 +128,7 @@ E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tes E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper @@ -161,7 +161,7 @@ E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tes E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt - + /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html index cb76abeacf..3c75ce390f 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html @@ -5,22 +5,34 @@ Job Report + + -

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

+

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

@@ -61,8 +73,29 @@ + + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html index 4353e38771..c2d0b95c97 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html @@ -5,28 +5,41 @@ Test Report + + -

Failed Test Cases on Your branch (Excludes Known Failure Cases)

Job Name
+

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- + + @@ -36,7 +49,8 @@ - + + @@ -44,7 +58,8 @@ - + + @@ -52,28 +67,17 @@ - + + - -
Test Case Test Script File Path Failure ReasonFailures across all other branches (20 latest testcases)Failures across all other branches (40 latest testcases)Dut Log URL Job URL Grafana URL
('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device components/driver/test_apps/i2c_test_apps/pytest_i2c.py failed on setup with "EOFError"0 / 200 / 40link link
esp32c3.release.test_esp_timer components/esp_timer/test_apps/pytest_esp_timer_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt0 / 200 / 40link link
esp32c3.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt0 / 200 / 40link link

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- - - - - - - - - - - - + + @@ -81,7 +85,8 @@ - + + @@ -89,7 +94,8 @@ - + + @@ -97,12 +103,14 @@ - + + -
Test CaseTest Script File PathFailure ReasonFailures across all other branches (20 latest testcases)Job URLGrafana URL
('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py failed on setup with "EOFError"3 / 203 / 40link link
esp32c2.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py AssertionError: Unity test failed3 / 203 / 40link link
esp32c3.512safe.test_wear_levelling components/wear_levelling/test_apps/pytest_wear_levelling.py pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt3 / 203 / 40link link
esp32c3.release.test_wear_levelling components/wear_levelling/test_apps/pytest_wear_levelling.py pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt3 / 203 / 40link link

Known Failure Cases

+

Known Failure Cases

@@ -142,7 +150,8 @@ -
Test Caselink

Skipped Test Cases

+

Skipped Test Cases

@@ -159,7 +168,8 @@ -
Test Caselink

Succeeded Test Cases

+

Succeeded Test Cases

@@ -228,8 +238,29 @@ + + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py index 551eaece9f..abdd8a7b64 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py @@ -46,6 +46,17 @@ class TestReportGeneration(unittest.TestCase): self.addCleanup(self.gitlab_patcher.stop) self.addCleanup(self.env_patcher.stop) self.addCleanup(self.failure_rate_patcher.stop) + self.addCleanup(self.cleanup_files) + + def cleanup_files(self) -> None: + files_to_delete = [ + self.target_test_report_generator.skipped_test_cases_report_file, + self.target_test_report_generator.succeeded_cases_report_file, + self.target_test_report_generator.failed_cases_report_file, + ] + for file_path in files_to_delete: + if os.path.exists(file_path): + os.remove(file_path) def load_test_and_job_reports(self) -> None: self.expected_target_test_report_html = load_file( @@ -62,9 +73,23 @@ class TestReportGeneration(unittest.TestCase): jobs = [GitlabJob.from_json_data(job_json, failure_rates.get(job_json['name'], {})) for job_json in json.loads(jobs_response_raw)['jobs']] test_cases = parse_testcases_from_filepattern(os.path.join(self.reports_sample_data_path, 'XUNIT_*.xml')) self.target_test_report_generator = TargetTestReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, title='Test Report', test_cases=test_cases) + project_id=123, + mr_iid=1, + pipeline_id=456, + job_id=0, + commit_id='cccc', + title='Test Report', + test_cases=test_cases + ) self.job_report_generator = JobReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, title='Job Report', jobs=jobs) + project_id=123, + mr_iid=1, + pipeline_id=456, + job_id=0, + commit_id='cccc', + title='Job Report', + jobs=jobs + ) self.target_test_report_generator._known_failure_cases_set = { '*.test_wpa_supplicant_ut', 'esp32c3.release.test_esp_timer', @@ -72,7 +97,7 @@ class TestReportGeneration(unittest.TestCase): } test_cases_failed = [tc for tc in test_cases if tc.is_failure] for index, tc in enumerate(test_cases_failed): - tc.latest_total_count = 20 + tc.latest_total_count = 40 if index % 3 == 0: tc.latest_failed_count = 0 else: diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py index c8252f58ad..79cd3f44d7 100644 --- a/tools/ci/dynamic_pipelines/utils.py +++ b/tools/ci/dynamic_pipelines/utils.py @@ -66,10 +66,14 @@ def load_known_failure_cases() -> t.Optional[t.Set[str]]: if not known_failures_file: return None try: - with open(known_failures_file) as f: + with open(known_failures_file, 'r') as f: file_content = f.read() - known_cases_list = re.sub(re.compile('#.*\n'), '', file_content).split() - return {case.strip() for case in known_cases_list} + + pattern = re.compile(r'^(.*?)\s+#\s+([A-Z]+)-\d+', re.MULTILINE) + matches = pattern.findall(file_content) + + known_cases_list = [match[0].strip() for match in matches] + return set(known_cases_list) except FileNotFoundError: return None @@ -111,7 +115,7 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: response = requests.post( f'{ci_dash_api_backend_host}/jobs/failure_ratio', headers={'Authorization': f'Bearer {token}'}, - json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_COMMIT_BRANCH', '')]}, + json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, ) if response.status_code != 200: print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') @@ -128,20 +132,20 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: return combined_jobs -def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase]) -> t.List[TestCase]: +def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], branches_filter: dict) -> t.List[TestCase]: """ Fetches info about failure rates of testcases using an API request to ci-dashboard-api. :param failed_testcases: The list of failed testcases models. + :param branches_filter: The filter to filter testcases by branch names. :return: A list of testcases with enriched with failure rates data. """ token = os.getenv('ESPCI_TOKEN', '') ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') + req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter} response = requests.post( f'{ci_dash_api_backend_host}/testcases/failure_ratio', headers={'Authorization': f'Bearer {token}'}, - json={'testcase_names': [testcase.name for testcase in failed_testcases], - 'exclude_branches': [os.getenv('CI_COMMIT_BRANCH', '')], - }, + json=req_json, ) if response.status_code != 200: print(f'Failed to fetch testcases failure rate data: {response.status_code} with error: {response.text}') @@ -166,3 +170,34 @@ def load_file(file_path: str) -> str: """ with open(file_path, 'r') as file: return file.read() + + +def format_permalink(s: str) -> str: + """ + Formats a given string into a permalink. + + :param s: The string to be formatted into a permalink. + :return: The formatted permalink as a string. + """ + end_index = s.find('(') + + if end_index != -1: + trimmed_string = s[:end_index].strip() + else: + trimmed_string = s.strip() + + formatted_string = trimmed_string.lower().replace(' ', '-') + + return formatted_string + + +def get_report_url(job_id: int, output_filepath: str) -> str: + """ + Generates the url of the path where the report will be stored in the job's artifacts . + + :param job_id: The job identifier used to construct the URL. + :param output_filepath: The path to the output file. + :return: The modified URL pointing to the job's artifacts. + """ + url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}'
Test Case