diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58f42a995a..fc0403e0c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,4 +30,5 @@ include: - '.gitlab/ci/integration_test.yml' - '.gitlab/ci/host-test.yml' - '.gitlab/ci/deploy.yml' + - '.gitlab/ci/post_deploy.yml' - '.gitlab/ci/test-win.yml' diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 04e2612f40..bd6293a671 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -395,6 +395,6 @@ test_idf_pytest_plugin: junit: XUNIT_RESULT.xml script: - cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator - - python -m unittest test_target_test_report_generator.py + - python -m unittest test_report_generator.py - cd ${IDF_PATH}/tools/ci/idf_pytest - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml new file mode 100644 index 0000000000..0c28b54751 --- /dev/null +++ b/.gitlab/ci/post_deploy.yml @@ -0,0 +1,12 @@ +generate_failed_jobs_report: + stage: post_deploy + tags: [build, shiny] + image: $ESP_ENV_IMAGE + when: always + artifacts: + expire_in: 1 week + when: always + paths: + - job_report.html + script: + - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index 5fff31c3f5..e661e3ded3 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -135,6 +135,8 @@ class TestCase: ci_job_url: t.Optional[str] = None ci_dashboard_url: t.Optional[str] = None dut_log_url: t.Optional[str] = None + latest_total_count: int = 0 + latest_failed_count: int = 0 @property def is_failure(self) -> bool: @@ -179,3 +181,45 @@ class TestCase: kwargs['skipped'] = skipped_node.attrib['message'] return cls(**kwargs) # type: ignore + + +@dataclass +class GitlabJob: + id: int + name: str + stage: str + status: str + url: str + ci_dashboard_url: str + failure_reason: t.Optional[str] = None + failure_log: t.Optional[str] = None + latest_total_count: int = 0 + latest_failed_count: int = 0 + + @property + def is_failed(self) -> bool: + return self.status == 'failed' + + @property + def is_success(self) -> bool: + return self.status == 'success' + + @classmethod + def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']: + grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list') + encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote) + + kwargs = { + 'id': job_data['id'], + 'name': job_data['name'], + 'stage': job_data['stage'], + 'status': job_data['status'], + 'url': job_data['url'], + 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', + 'failure_reason': job_data['failure_reason'], + 'failure_log': job_data['failure_log'], + 'latest_total_count': failure_data.get('total_count', 0), + 'latest_failed_count': failure_data.get('failed_count', 0), + } + + return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index 1f0de199aa..dd3951c107 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -18,7 +18,9 @@ from prettytable import PrettyTable from .constants import COMMENT_START_MARKER from .constants import REPORT_TEMPLATE_FILEPATH from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME +from .models import GitlabJob from .models import TestCase +from .utils import fetch_failed_testcases_failure_ratio from .utils import is_url from .utils import load_known_failure_cases @@ -58,25 +60,150 @@ class ReportGenerator: def table_to_html_str(table: PrettyTable) -> str: return html.unescape(table.get_html_string()) # type: ignore + def create_table_section( + self, + report_sections: list, + title: str, + items: list, + headers: list, + row_attrs: list, + value_functions: t.Optional[list] = None, + ) -> None: + """ + Appends a formatted section to a report based on the provided items. This section includes + a header and a table constructed from the items list with specified headers and attributes. + + :param report_sections: List where the HTML report sections are collected. This list is + modified in-place by appending new sections. + :param title: Title for the report section. This title is used as a header above the table. + :param items: List of item objects to include in the table. Each item should have attributes + that correspond to the row_attrs and value_functions specified. + :param headers: List of strings that will serve as the column headers in the generated table. + :param row_attrs: List of attributes to include from each item for the table rows. These + should be attributes or keys that exist on the items in the 'items' list. + :param value_functions: Optional list of tuples containing additional header and corresponding + value function. Each tuple should specify a header (as a string) and + a function that takes an item and returns a string. This is used for + generating dynamic columns based on item data. + + :return: None. The function modifies the 'report_sections' list by appending new HTML sections. + """ + if not items: + return + + report_sections.append(f'

{title}

') + report_sections.append( + self._create_table_for_items( + items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or [] + ) + ) + + def _create_table_for_items( + self, + items: t.Union[t.List[TestCase], t.List[GitlabJob]], + headers: t.List[str], + row_attrs: t.List[str], + value_functions: t.Optional[t.List[t.Tuple[str, t.Callable[[t.Union[TestCase, GitlabJob]], str]]]] = None, + ) -> str: + """ + Create a PrettyTable and convert it to an HTML string for the provided test cases. + :param items: List of item objects to include in the table. + :param headers: List of strings for the table headers. + :param row_attrs: List of attributes to include in each row. + :param value_functions: List of tuples containing additional header and corresponding value function. + :return: HTML table string. + """ + table = PrettyTable() + table.field_names = headers + + # Create a mapping of header names to their corresponding index in the headers list + header_index_map = {header: i for i, header in enumerate(headers)} + + for item in items: + row = [] + for attr in row_attrs: + value = str(getattr(item, attr, '')) + if is_url(value): + link = f'link' + row.append(link) + else: + row.append(value) + + # Insert values computed by value functions at the correct column position based on their headers + if value_functions: + for header, func in value_functions: + index = header_index_map.get(header) + if index is not None: + computed_value = func(item) + row.insert(index, computed_value) + + table.add_row(row) + + return self.table_to_html_str(table) + + @staticmethod + def _filter_items( + items: t.Union[t.List[TestCase], t.List[GitlabJob]], condition: t.Callable[[t.Union[TestCase, GitlabJob]], bool] + ) -> t.List[TestCase]: + """ + Filter items s based on a given condition. + + :param items: List of items to filter by given condition. + :param condition: A function that evaluates to True or False for each items. + :return: List of filtered instances. + """ + return [item for item in items if condition(item)] + + @staticmethod + def _sort_items( + items: t.List[t.Union[TestCase, GitlabJob]], + key: t.Union[str, t.Callable[[t.Union[TestCase, GitlabJob]], t.Any]], + order: str = 'asc', + ) -> t.List[t.Union[TestCase, GitlabJob]]: + """ + Sort items based on a given key and order. + + :param items: List of items to sort. + :param key: A string representing the attribute name or a function to extract the sorting key. + :param order: Order of sorting ('asc' for ascending, 'desc' for descending). + :return: List of sorted instances. + """ + key_func = None + if isinstance(key, str): + + def key_func(item: t.Any) -> t.Any: + return getattr(item, key) + + if key_func is not None: + try: + items = sorted(items, key=key_func, reverse=(order == 'desc')) + except TypeError: + print(f'Comparison for the key {key} is not supported') + return items + @abc.abstractmethod def _get_report_str(self) -> str: raise NotImplementedError def post_report(self, job_id: int, commit_id: str) -> None: # report in html format, otherwise will exceed the limit - with open(self.output_filepath, 'w') as fw: - fw.write(self._get_report_str()) - - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt - # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') - comment = f'#### {self.title}\n' + + report_str = self._get_report_str() + if self.additional_info: comment += f'{self.additional_info}\n' - comment += f""" -Full {self.title} here: {url}/-/jobs/{job_id}/artifacts/{self.output_filepath} (with commit {commit_id}) + if report_str: + with open(self.output_filepath, 'w') as fw: + fw.write(report_str) + + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` + url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + + comment += f""" +Full {self.title} here: {url}/-/jobs/{job_id}/artifacts/{self.output_filepath} (with commit {commit_id[:8]}) """ print(comment) @@ -265,94 +392,172 @@ class TargetTestReportGenerator(ReportGenerator): ] return matched_cases - def _filter_test_cases(self, condition: t.Callable[[TestCase], bool]) -> t.List[TestCase]: - """ - Filter test cases based on a given condition. In this scenario, we filter by status, - however it is possible to filter by other criteria. - - :param condition: A function that evaluates to True or False for each test case. - :return: List of filtered TestCase instances. - """ - return [tc for tc in self.test_cases if condition(tc)] - - def _create_table_for_test_cases( - self, test_cases: t.List[TestCase], headers: t.List[str], row_attrs: t.List[str] - ) -> str: - """ - Create a PrettyTable and convert it to an HTML string for the provided test cases. - :param test_cases: List of TestCase objects to include in the table. - :param headers: List of strings for the table headers. - :param row_attrs: List of attributes to include in each row. - :return: HTML table string. - """ - table = PrettyTable() - table.field_names = headers - for tc in test_cases: - row = [] - for attr in row_attrs: - value = getattr(tc, attr, '') - if is_url(value): - link = f'link' - row.append(link) - else: - row.append(value) - table.add_row(row) - - return self.table_to_html_str(table) - def _get_report_str(self) -> str: """ Generate a complete HTML report string by processing test cases. :return: Complete HTML report string. """ - table_str = '' + report_parts: list = [] known_failures = self.get_known_failure_cases() known_failure_case_names = {case.name for case in known_failures} - failed_test_cases = self._filter_test_cases( - lambda tc: tc.is_failure and tc.name not in known_failure_case_names + failed_test_cases = self._filter_items( + self.test_cases, lambda tc: tc.is_failure and tc.name not in known_failure_case_names ) - skipped_test_cases = self._filter_test_cases(lambda tc: tc.is_skipped) - successful_test_cases = self._filter_test_cases(lambda tc: tc.is_success) + failed_test_cases_with_ratio = self._sort_items( + fetch_failed_testcases_failure_ratio(failed_test_cases), key='latest_failed_count' + ) + skipped_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_skipped) + successful_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_success) - if failed_test_cases: - table_str += '

Failed Test Cases (Excludes Known Failure Cases)

' - table_str += self._create_table_for_test_cases( - test_cases=failed_test_cases, - headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - ) + current_branch_failures = self._sort_items( + self._filter_items(failed_test_cases_with_ratio, lambda tc: tc.latest_failed_count == 0), + key='latest_failed_count', + ) + other_branch_failures = self._sort_items( + self._filter_items( + failed_test_cases_with_ratio, lambda tc: tc.name not in [t.name for t in current_branch_failures] + ), + key='latest_failed_count', + ) - if known_failures: - table_str += '

Known Failure Cases

' - table_str += self._create_table_for_test_cases( - test_cases=known_failures, - headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - ) + self.create_table_section( + report_sections=report_parts, + title='Failed Test Cases on Your branch (Excludes Known Failure Cases)', + items=current_branch_failures, + headers=[ + 'Test Case', + 'Test Script File Path', + 'Failure Reason', + 'Failures across all other branches (20 latest testcases)', + 'Job URL', + 'Grafana URL', + ], + row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures across all other branches (20 latest testcases)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) + self.create_table_section( + report_sections=report_parts, + title='Failed Test Cases on Other branches (Excludes Known Failure Cases)', + items=other_branch_failures, + headers=[ + 'Test Case', + 'Test Script File Path', + 'Failure Reason', + 'Failures across all other branches (20 latest testcases)', + 'Job URL', + 'Grafana URL', + ], + row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures across all other branches (20 latest testcases)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) - if skipped_test_cases: - table_str += '

Skipped Test Cases

' - table_str += self._create_table_for_test_cases( - test_cases=skipped_test_cases, - headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], - row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], - ) + self.create_table_section( + report_sections=report_parts, + title='Known Failure Cases', + items=known_failures, + headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], + ) + self.create_table_section( + report_sections=report_parts, + title='Skipped Test Cases', + items=skipped_test_cases, + headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], + row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], + ) + self.create_table_section( + report_sections=report_parts, + title='Succeeded Test Cases', + items=successful_test_cases, + headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], + row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], + ) - if successful_test_cases: - table_str += '

Succeeded Test Cases

' - table_str += self._create_table_for_test_cases( - test_cases=successful_test_cases, - headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], - ) self.additional_info = ( '**Test Case Summary:**\n' - f'- **Failed Test Cases (Excludes Known Failure Cases):** {len(failed_test_cases)}\n' + f'- **Failed Test Cases on Your Branch (Excludes Known Failure Cases):** {len(current_branch_failures)}.\n' + f'- **Failed Test Cases on Other Branches (Excludes Known Failure Cases):** {len(other_branch_failures)}.\n' f'- **Known Failures:** {len(known_failures)}\n' f'- **Skipped Test Cases:** {len(skipped_test_cases)}\n' f'- **Succeeded Test Cases:** {len(successful_test_cases)}\n\n' - f'Please check report below for more information.\n\n' + 'Please check report below for more information.\n\n' ) - return self.generate_html_report(table_str) + return self.generate_html_report(''.join(report_parts)) + + +class JobReportGenerator(ReportGenerator): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + *, + title: str = 'Job Report', + jobs: t.List[GitlabJob], + ): + super().__init__(project_id, mr_iid, pipeline_id, title=title) + self.jobs = jobs + + def _get_report_str(self) -> str: + """ + Generate a complete HTML report string by processing jobs. + :return: Complete HTML report string. + """ + report_str: str = '' + report_parts: list = [] + + if not self.jobs: + print('No jobs found, skip generating job report') + return 'No Job Found' + + relevant_failed_jobs = self._sort_items( + self._filter_items( + self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test'] + ), + key='latest_failed_count', + ) + succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) + + self.additional_info = ( + '**Job Summary:**\n' + f'- **Failed Jobs (Excludes "integration_test" and "target_test" jobs):** {len(relevant_failed_jobs)}\n' + f'- **Succeeded Jobs:** {len(succeeded_jobs)}\n\n' + ) + + if relevant_failed_jobs: + self.create_table_section( + report_sections=report_parts, + title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', + items=relevant_failed_jobs, + headers=[ + 'Job Name', + 'Failure Reason', + 'Failure Log', + 'Failures across all other branches (10 latest jobs)', + 'URL', + 'CI Dashboard URL', + ], + row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], + value_functions=[ + ( + 'Failures across all other branches (10 latest jobs)', + lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", + ) + ], + ) + self.additional_info += f'Please check report below for more information.\n\n' + report_str = self.generate_html_report(''.join(report_parts)) + + return report_str diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_report.py b/tools/ci/dynamic_pipelines/scripts/generate_build_report.py deleted file mode 100644 index cef16271fe..0000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_report.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import glob -import os - -import __init__ # noqa: F401 # inject the system path -from dynamic_pipelines.report import BuildReportGenerator -from idf_ci.app import import_apps_from_txt - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Update Build Report in MR pipelines', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - '--project-id', - type=int, - default=os.getenv('CI_PROJECT_ID'), - help='Project ID', - ) - parser.add_argument( - '--mr-iid', - type=int, - default=os.getenv('CI_MERGE_REQUEST_IID'), - help='Merge Request IID', - ) - parser.add_argument( - '--pipeline-id', - type=int, - default=os.getenv('PARENT_PIPELINE_ID'), - help='Pipeline ID', - ) - parser.add_argument( - '--job-id', - type=int, - default=os.getenv('CI_JOB_ID'), - help='Job ID', - ) - parser.add_argument( - '--commit-id', - default=os.getenv('CI_COMMIT_SHORT_SHA'), - help='MR commit ID', - ) - parser.add_argument( - '--app-list-filepattern', - default='list_job_*.txt', - help='App list file pattern', - ) - - args = parser.parse_args() - - apps = [] - for f in glob.glob(args.app_list_filepattern): - apps.extend(import_apps_from_txt(f)) - - report_generator = BuildReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, apps=apps) - report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py new file mode 100644 index 0000000000..bb2996d191 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_report.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import glob +import os +import typing as t + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.report import BuildReportGenerator +from dynamic_pipelines.report import JobReportGenerator +from dynamic_pipelines.report import TargetTestReportGenerator +from dynamic_pipelines.utils import fetch_failed_jobs +from dynamic_pipelines.utils import parse_testcases_from_filepattern +from idf_ci.app import import_apps_from_txt + + +def main() -> None: + parser: argparse.ArgumentParser = setup_argument_parser() + args: argparse.Namespace = parser.parse_args() + + report_actions: t.Dict[str, t.Callable[[argparse.Namespace], None]] = { + 'build': generate_build_report, + 'target_test': generate_target_test_report, + 'job': generate_jobs_report, + } + + report_action = report_actions.get(args.report_type) + if report_action is None: + raise ValueError('Unknown report type is requested to be generated.') + + report_action(args) + + +def setup_argument_parser() -> argparse.ArgumentParser: + report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False) + report_type_parser.add_argument( + '--report-type', choices=['build', 'target_test', 'job'], required=True, help='Type of report to generate' + ) + report_type_args: argparse.Namespace + remaining_args: t.List[str] + report_type_args, remaining_args = report_type_parser.parse_known_args() + + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description='Update reports in MR pipelines based on the selected report type', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + parents=[report_type_parser], + ) + + common_arguments(parser) + conditional_arguments(report_type_args, parser) + + return parser + + +def common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID') + parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID') + parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID') + parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID') + parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA'), help='MR commit ID') + + +def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: + if report_type_args.report_type == 'build': + parser.add_argument('--app-list-filepattern', default='list_job_*.txt', help='Pattern to match app list files') + elif report_type_args.report_type == 'target_test': + parser.add_argument( + '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' + ) + + +def generate_build_report(args: argparse.Namespace) -> None: + apps: t.List[t.Any] = [ + app for file_name in glob.glob(args.app_list_filepattern) for app in import_apps_from_txt(file_name) + ] + report_generator = BuildReportGenerator( + args.project_id, args.mr_iid, args.pipeline_id, apps=apps + ) + report_generator.post_report(args.job_id, args.commit_id) + + +def generate_target_test_report(args: argparse.Namespace) -> None: + test_cases: t.List[t.Any] = parse_testcases_from_filepattern(args.junit_report_filepattern) + report_generator = TargetTestReportGenerator( + args.project_id, args.mr_iid, args.pipeline_id, test_cases=test_cases + ) + report_generator.post_report(args.job_id, args.commit_id) + + +def generate_jobs_report(args: argparse.Namespace) -> None: + jobs: t.List[t.Any] = fetch_failed_jobs(args.commit_id) + + if not jobs: + return + + report_generator = JobReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, jobs=jobs) + report_generator.post_report(args.job_id, args.commit_id) + + +if __name__ == '__main__': + main() diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py deleted file mode 100644 index 07a8952bcf..0000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os - -import __init__ # noqa: F401 # inject the system path -from dynamic_pipelines.report import TargetTestReportGenerator -from dynamic_pipelines.utils import parse_testcases_from_filepattern - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Update Build Report in MR pipelines', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - '--project-id', - type=int, - default=os.getenv('CI_PROJECT_ID'), - help='Project ID', - ) - parser.add_argument( - '--mr-iid', - type=int, - default=os.getenv('CI_MERGE_REQUEST_IID'), - help='Merge Request IID', - ) - parser.add_argument( - '--pipeline-id', - type=int, - default=os.getenv('PARENT_PIPELINE_ID'), - help='Pipeline ID', - ) - parser.add_argument( - '--job-id', - type=int, - default=os.getenv('CI_JOB_ID'), - help='Job ID', - ) - parser.add_argument( - '--commit-id', - default=os.getenv('CI_COMMIT_SHORT_SHA'), - help='MR commit ID', - ) - parser.add_argument( - '--junit-report-filepattern', - default='XUNIT_RESULT*.xml', - help='Junit Report file pattern', - ) - - args = parser.parse_args() - - test_cases = parse_testcases_from_filepattern(args.junit_report_filepattern) - report_generator = TargetTestReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, test_cases=test_cases) - report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml index 6559bc3502..8fe17af72e 100644 --- a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml +++ b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml @@ -8,4 +8,4 @@ generate_pytest_report: - target_test_report.html script: - python tools/ci/get_known_failure_cases_file.py - - python tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py + - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html index f8da79ff7e..6997fa45c1 100644 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -1,23 +1,57 @@ - + {{title}} - - - - - - + + + + +
{{table}}
- + + diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 1b531790eb..336b74f0a6 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -10,7 +10,7 @@ generate_pytest_build_report: - build_report.html - test_related_apps_download_urls.yml script: - - python tools/ci/dynamic_pipelines/scripts/generate_build_report.py + - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html new file mode 100644 index 0000000000..cb76abeacf --- /dev/null +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html @@ -0,0 +1,94 @@ + + + + + Job Report + + + + + +

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Job NameFailure ReasonFailure LogFailures across all other branches (10 latest jobs)URLCI Dashboard URL
build_clang_test_apps_esp32h2Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason2 / 10linklink
build_template_appSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason3 / 10linklink
check_public_headersSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Log4 / 10linklink
+ + + + + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html index b4c143879b..4353e38771 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html @@ -1,26 +1,32 @@ - + Test Report - - - - - - -

Failed Test Cases (Excludes Known Failure Cases)

+ + + + + +

Failed Test Cases on Your branch (Excludes Known Failure Cases)

+ @@ -30,20 +36,68 @@ + + + + + + + + + + + + + + + + + + +
Test Case Test Script File Path Failure ReasonFailures across all other branches (20 latest testcases) Job URL Grafana URL
('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device components/driver/test_apps/i2c_test_apps/pytest_i2c.py failed on setup with "EOFError"0 / 20 link
esp32c3.release.test_esp_timercomponents/esp_timer/test_apps/pytest_esp_timer_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt0 / 20link
esp32c3.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt0 / 20link

Failed Test Cases on Other branches (Excludes Known Failure Cases)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -174,5 +228,34 @@ - + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json new file mode 100644 index 0000000000..18e4bea1e2 --- /dev/null +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json @@ -0,0 +1,7 @@ +{ + "jobs": [ + {"failed_count": 2, "failure_ratio": 0.2, "total_count": 10, "name": "build_clang_test_apps_esp32h2"}, + {"failed_count": 3, "failure_ratio": 0.3, "total_count": 10, "name": "build_template_app"}, + {"failed_count": 4, "failure_ratio": 0.4, "total_count": 10, "name": "check_public_headers"} + ] +} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json new file mode 100644 index 0000000000..844e2545b5 --- /dev/null +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json @@ -0,0 +1,212 @@ +{ + "jobs": [ + { + "duration_sec": 42.158688, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:38:16, 24 May 2024", + "id": 48838677, + "name": "check_pre_commit", + "pending_sec": 1.15148, + "runner_name": "FA002598-build", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838677" + }, + { + "duration_sec": 35.438477, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:37:32, 24 May 2024", + "id": 48838675, + "name": "run-danger-mr-linter", + "pending_sec": 0.371668, + "runner_name": "BrnoVM0211", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838675" + }, + { + "duration_sec": 30.202475, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:37:28, 24 May 2024", + "id": 48838682, + "name": "check_esp_system", + "pending_sec": 1.148756, + "runner_name": "ruby6-cent9 [32]", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838682" + }, + { + "duration_sec": 33.75121, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:37:31, 24 May 2024", + "id": 48838679, + "name": "check_blobs", + "pending_sec": 0.725292, + "runner_name": "gem3-cent9 [32]", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838679" + }, + { + "duration_sec": 121.84324, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:41:35, 24 May 2024", + "id": 48838687, + "name": "code_quality_check", + "pending_sec": 0.271973, + "runner_name": "ruby15-cent9 [32]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838687" + }, + { + "duration_sec": 153.68849, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:39:32, 24 May 2024", + "id": 48838686, + "name": "fast_template_app", + "pending_sec": 2.319577, + "runner_name": "FA002598-build", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838686" + }, + { + "duration_sec": 25.572954, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:37:23, 24 May 2024", + "id": 48838684, + "name": "check_configure_ci_environment_parsing", + "pending_sec": 1.184287, + "runner_name": "gem3-cent9 [32]", + "stage": "pre_check", + "status": "success", + "url": "https://test.com/-/jobs/48838684" + }, + { + "duration_sec": 120.95287, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:41:34, 24 May 2024", + "id": 48838690, + "name": "build_clang_test_apps_esp32s3", + "pending_sec": 0.671956, + "runner_name": "ruby7-cent9 [32]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838690" + }, + { + "duration_sec": 165.74513, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:42:19, 24 May 2024", + "id": 48838692, + "name": "build_clang_test_apps_esp32c2", + "pending_sec": 0.82007, + "runner_name": "PowerfulBuildRunner03 [16]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838692" + }, + { + "duration_sec": 95.72326, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:41:09, 24 May 2024", + "id": 48838696, + "name": "build_clang_test_apps_esp32p4", + "pending_sec": 0.567116, + "runner_name": "gem3-cent9 [32]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838696" + }, + { + "duration_sec": 122.19848, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:41:36, 24 May 2024", + "id": 48838691, + "name": "build_clang_test_apps_esp32c3", + "pending_sec": 0.709112, + "runner_name": "ruby6-cent9 [32]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838691" + }, + { + "duration_sec": 148.09895, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:42:02, 24 May 2024", + "id": 48838694, + "name": "build_clang_test_apps_esp32c5", + "pending_sec": 0.779584, + "runner_name": "PowerfulBuildRunner04 [15]", + "stage": "build", + "status": "success", + "url": "https://test.com/-/jobs/48838694" + }, + { + "duration_sec": 20.275927, + "failure_log": null, + "failure_reason": null, + "finished_at": "03:39:54, 24 May 2024", + "id": 48838699, + "name": "gen_integration_pipeline", + "pending_sec": 0.868898, + "runner_name": "FA002598-build", + "stage": "assign_test", + "status": "success", + "url": "https://test.com/-/jobs/48838699" + }, + { + "duration_sec": 103.08849, + "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", + "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", + "finished_at": "03:41:17, 24 May 2024", + "id": 48838695, + "name": "build_clang_test_apps_esp32h2", + "pending_sec": 0.765111, + "runner_name": "gem2-cent9 [32]", + "stage": "build", + "status": "failed", + "url": "https://test.com/-/jobs/48838695" + }, + { + "duration_sec": 634.59467, + "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", + "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", + "finished_at": "03:50:09, 24 May 2024", + "id": 48838704, + "name": "build_template_app", + "pending_sec": 0.161796, + "runner_name": "ruby6-cent9 [32]", + "stage": "host_test", + "status": "failed", + "url": "https://test.com/-/jobs/48838704" + }, + { + "duration_sec": 1060.0835, + "failure_log": "Some Failure Log", + "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", + "finished_at": "03:55:14, 24 May 2024", + "id": 48838705, + "name": "check_public_headers", + "pending_sec": 0.449408, + "runner_name": "ruby6-cent9 [32]", + "stage": "host_test", + "status": "failed", + "url": "https://test.com/-/jobs/48838705" + } + ] +} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py new file mode 100644 index 0000000000..551eaece9f --- /dev/null +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import json +import os.path +import sys +import unittest +from unittest.mock import MagicMock +from unittest.mock import patch + +sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci', 'python_packages')) +sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci')) + +from dynamic_pipelines.models import GitlabJob # noqa: E402 +from dynamic_pipelines.report import JobReportGenerator, TargetTestReportGenerator # noqa: E402 +from dynamic_pipelines.utils import load_file, parse_testcases_from_filepattern # noqa: E402 + + +class TestReportGeneration(unittest.TestCase): + def setUp(self) -> None: + self.reports_sample_data_path = os.path.join( + os.environ.get('IDF_PATH', ''), 'tools', 'ci', 'dynamic_pipelines', 'tests', 'test_report_generator', 'reports_sample_data' + ) + self.setup_patches() + self.load_test_and_job_reports() + self.create_report_generators() + + def setup_patches(self) -> None: + self.gitlab_patcher = patch('dynamic_pipelines.report.Gitlab') + self.failure_rate_patcher = patch('dynamic_pipelines.report.fetch_failed_testcases_failure_ratio') + self.env_patcher = patch.dict('os.environ', { + 'CI_DASHBOARD_HOST': 'https://test_dashboard_host', + 'CI_PAGES_URL': 'https://artifacts_path', + 'CI_JOB_ID': '1', + }) + + self.MockGitlab = self.gitlab_patcher.start() + self.test_cases_failure_rate = self.failure_rate_patcher.start() + self.env_patcher.start() + + self.mock_project = MagicMock() + self.mock_mr = MagicMock() + self.MockGitlab.return_value.project = self.mock_project + self.mock_project.mergerequests.get.return_value = self.mock_mr + + self.addCleanup(self.gitlab_patcher.stop) + self.addCleanup(self.env_patcher.stop) + self.addCleanup(self.failure_rate_patcher.stop) + + def load_test_and_job_reports(self) -> None: + self.expected_target_test_report_html = load_file( + os.path.join(self.reports_sample_data_path, 'expected_target_test_report.html') + ) + self.expected_job_report_html = load_file( + os.path.join(self.reports_sample_data_path, 'expected_job_report.html') + ) + + def create_report_generators(self) -> None: + jobs_response_raw = load_file(os.path.join(self.reports_sample_data_path, 'jobs_api_response.json')) + failure_rate_jobs_response = load_file(os.path.join(self.reports_sample_data_path, 'failure_rate_jobs_response.json')) + failure_rates = {item['name']: item for item in json.loads(failure_rate_jobs_response).get('jobs', [])} + jobs = [GitlabJob.from_json_data(job_json, failure_rates.get(job_json['name'], {})) for job_json in json.loads(jobs_response_raw)['jobs']] + test_cases = parse_testcases_from_filepattern(os.path.join(self.reports_sample_data_path, 'XUNIT_*.xml')) + self.target_test_report_generator = TargetTestReportGenerator( + project_id=123, mr_iid=1, pipeline_id=456, title='Test Report', test_cases=test_cases) + self.job_report_generator = JobReportGenerator( + project_id=123, mr_iid=1, pipeline_id=456, title='Job Report', jobs=jobs) + self.target_test_report_generator._known_failure_cases_set = { + '*.test_wpa_supplicant_ut', + 'esp32c3.release.test_esp_timer', + '*.512safe.test_wear_levelling', + } + test_cases_failed = [tc for tc in test_cases if tc.is_failure] + for index, tc in enumerate(test_cases_failed): + tc.latest_total_count = 20 + if index % 3 == 0: + tc.latest_failed_count = 0 + else: + tc.latest_failed_count = 3 + self.test_cases_failure_rate.return_value = test_cases_failed + + def test_known_failure_cases(self) -> None: + known_failure_cases = self.target_test_report_generator.get_known_failure_cases() + self.assertEqual(len(known_failure_cases), 4) + + def test_failed_cases_in_target_test_report(self) -> None: + known_failures = self.target_test_report_generator.get_known_failure_cases() + known_failure_case_names = {case.name for case in known_failures} + failed_testcases = self.target_test_report_generator._filter_items( + self.target_test_report_generator.test_cases, + lambda tc: tc.is_failure and tc.name not in known_failure_case_names, + ) + self.assertEqual(len(failed_testcases), 3) + + def test_skipped_cases_in_target_test_report(self) -> None: + skipped_testcases = self.target_test_report_generator._filter_items( + self.target_test_report_generator.test_cases, lambda tc: tc.is_skipped + ) + self.assertEqual(len(skipped_testcases), 1) + + def test_successful_cases_in_target_test_report(self) -> None: + succeeded_testcases = self.target_test_report_generator._filter_items( + self.target_test_report_generator.test_cases, lambda tc: tc.is_success + ) + self.assertEqual(len(succeeded_testcases), 9) + + def test_target_test_report_html_structure(self) -> None: + report = self.target_test_report_generator._get_report_str() + self.assertEqual(report, self.expected_target_test_report_html) + + def test_failed_jobs_in_job_report(self) -> None: + failed_jobs = self.job_report_generator._filter_items(self.job_report_generator.jobs, lambda job: job.is_failed) + self.assertEqual(len(failed_jobs), 3) + + def test_successful_jobs_in_job_report(self) -> None: + succeeded_jobs = self.job_report_generator._filter_items( + self.job_report_generator.jobs, lambda job: job.is_success + ) + self.assertEqual(len(succeeded_jobs), 13) + + def test_job_report_html_structure(self) -> None: + report = self.job_report_generator._get_report_str() + self.assertEqual(report, self.expected_job_report_html) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_target_test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_target_test_report_generator.py deleted file mode 100644 index a3d517e3dc..0000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_target_test_report_generator.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os.path -import sys -import unittest -from unittest.mock import MagicMock -from unittest.mock import patch - -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci', 'python_packages')) -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci')) - - -from dynamic_pipelines.report import TargetTestReportGenerator # noqa: E402 -from dynamic_pipelines.utils import parse_testcases_from_filepattern # noqa: E402 - - -class TestReportGeneration(unittest.TestCase): - @classmethod - def load_expected_report(cls, file_path: str) -> str: - """ - Loads the content of an expected report HTML file. - - :param file_path: The path to the file containing the expected HTML report. - :return: The content of the file as a string. - """ - with open(file_path, 'r') as file: - return file.read() - - def setUp(self) -> None: - patcher = patch('dynamic_pipelines.report.Gitlab') - env_patcher = patch.dict('os.environ', { - 'CI_DASHBOARD_HOST': 'https://test_dashboard_host', - 'CI_PAGES_URL': 'https://artifacts_path', - 'CI_JOB_ID': '1', - }) - env_patcher.start() - self.MockGitlab = patcher.start() - self.addCleanup(patcher.stop) - self.addCleanup(env_patcher.stop) - self.reports_sample_data_path = os.path.join( - os.environ.get('IDF_PATH', ''), # type: ignore - 'tools', - 'ci', - 'dynamic_pipelines', - 'tests', - 'test_report_generator', - 'reports_sample_data' - ) - self.mock_project = MagicMock() - self.mock_mr = MagicMock() - - self.MockGitlab.return_value.project = self.mock_project - self.mock_project.mergerequests.get.return_value = self.mock_mr - - self.expected_report_html = self.load_expected_report( - os.path.join(self.reports_sample_data_path, 'expected_target_test_report.html') - ) - - test_cases = parse_testcases_from_filepattern(os.path.join(self.reports_sample_data_path, 'XUNIT_*.xml')) - self.report_generator = TargetTestReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, title='Test Report', test_cases=test_cases - ) - self.report_generator._known_failure_cases_set = { - '*.test_wpa_supplicant_ut', - 'esp32c3.release.test_esp_timer', - '*.512safe.test_wear_levelling', - } - - def test_known_failure_cases(self) -> None: - known_failure_cases = self.report_generator.get_known_failure_cases() - self.assertEqual(len(known_failure_cases), 4) - - def test_failed_cases_in_report(self) -> None: - known_failures = self.report_generator.get_known_failure_cases() - known_failure_case_names = {case.name for case in known_failures} - failed_testcases = self.report_generator._filter_test_cases( - lambda tc: tc.is_failure and tc.name not in known_failure_case_names - ) - self.assertEqual(len(failed_testcases), 3) - - def test_skipped_cases_in_report(self) -> None: - skipped_testcases = self.report_generator._filter_test_cases(lambda tc: tc.is_skipped) - self.assertEqual(len(skipped_testcases), 1) - - def test_successful_cases_in_report(self) -> None: - succeeded_testcases = self.report_generator._filter_test_cases(lambda tc: tc.is_success) - self.assertEqual(len(succeeded_testcases), 9) - - def test_complete_html_structure(self) -> None: - report = self.report_generator._get_report_str() - self.assertEqual(report, self.expected_report_html) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py index d10bfce50a..c8252f58ad 100644 --- a/tools/ci/dynamic_pipelines/utils.py +++ b/tools/ci/dynamic_pipelines/utils.py @@ -7,8 +7,10 @@ import typing as t import xml.etree.ElementTree as ET from urllib.parse import urlparse +import requests import yaml +from .models import GitlabJob from .models import Job from .models import TestCase @@ -81,3 +83,86 @@ def is_url(string: str) -> bool: """ parsed = urlparse(string) return bool(parsed.scheme) and bool(parsed.netloc) + + +def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: + """ + Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api. + :param commit_id: The commit ID for which to fetch jobs. + :return: A list of jobs if the request is successful, otherwise an empty list. + """ + token = os.getenv('ESPCI_TOKEN', '') + ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') + response = requests.get( + f'{ci_dash_api_backend_host}/commits/{commit_id}/jobs', + headers={'Authorization': f'Bearer {token}'} + ) + if response.status_code != 200: + print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') + return [] + + data = response.json() + jobs = data.get('jobs', []) + + if not jobs: + return [] + + failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] + response = requests.post( + f'{ci_dash_api_backend_host}/jobs/failure_ratio', + headers={'Authorization': f'Bearer {token}'}, + json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_COMMIT_BRANCH', '')]}, + ) + if response.status_code != 200: + print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') + return [] + + failure_rate_data = response.json() + failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])} + + combined_jobs = [] + for job in jobs: + failure_data = failure_rates.get(job['name'], {}) + combined_jobs.append(GitlabJob.from_json_data(job, failure_data)) + + return combined_jobs + + +def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase]) -> t.List[TestCase]: + """ + Fetches info about failure rates of testcases using an API request to ci-dashboard-api. + :param failed_testcases: The list of failed testcases models. + :return: A list of testcases with enriched with failure rates data. + """ + token = os.getenv('ESPCI_TOKEN', '') + ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') + response = requests.post( + f'{ci_dash_api_backend_host}/testcases/failure_ratio', + headers={'Authorization': f'Bearer {token}'}, + json={'testcase_names': [testcase.name for testcase in failed_testcases], + 'exclude_branches': [os.getenv('CI_COMMIT_BRANCH', '')], + }, + ) + if response.status_code != 200: + print(f'Failed to fetch testcases failure rate data: {response.status_code} with error: {response.text}') + return [] + + failure_rate_data = response.json() + failure_rates = {item['name']: item for item in failure_rate_data.get('testcases', [])} + + for testcase in failed_testcases: + testcase.latest_total_count = failure_rates.get(testcase.name, {}).get('total_count', 0) + testcase.latest_failed_count = failure_rates.get(testcase.name, {}).get('failed_count', 0) + + return failed_testcases + + +def load_file(file_path: str) -> str: + """ + Loads the content of a file. + + :param file_path: The path to the file needs to be loaded. + :return: The content of the file as a string. + """ + with open(file_path, 'r') as file: + return file.read()
Test CaseTest Script File PathFailure ReasonFailures across all other branches (20 latest testcases)Job URLGrafana URL
('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev components/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.py failed on setup with "EOFError"3 / 20 link
esp32c2.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pyAssertionError: Unity test failed3 / 20link
esp32c3.512safe.test_wear_levellingcomponents/wear_levelling/test_apps/pytest_wear_levelling.pypexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt3 / 20link
esp32c3.release.test_wear_levelling components/wear_levelling/test_apps/pytest_wear_levelling.py pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt3 / 20 link