diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc0403e0c4..0e449ec7e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,4 +31,5 @@ include: - '.gitlab/ci/host-test.yml' - '.gitlab/ci/deploy.yml' - '.gitlab/ci/post_deploy.yml' + - '.gitlab/ci/retry_failed_jobs.yml' - '.gitlab/ci/test-win.yml' diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index b43e1d3d57..223a1f9dd9 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -12,6 +12,7 @@ stages: - test_deploy - deploy - post_deploy + - retry_failed_jobs variables: # System environment diff --git a/.gitlab/ci/retry_failed_jobs.yml b/.gitlab/ci/retry_failed_jobs.yml new file mode 100644 index 0000000000..28a2c1e06f --- /dev/null +++ b/.gitlab/ci/retry_failed_jobs.yml @@ -0,0 +1,14 @@ +retry_failed_jobs: + stage: retry_failed_jobs + tags: [shiny, fast_run] + image: $ESP_ENV_IMAGE + dependencies: null + before_script: [] + cache: [] + extends: [] + script: + - echo "Retrieving and retrying all failed jobs for the pipeline..." + - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID + when: manual + needs: + - generate_failed_jobs_report diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 21f903338d..bbeefcfc4d 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -29,8 +29,18 @@ REPORT_TEMPLATE_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' ) +RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png' +RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:' +RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})' + BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) + +CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '') +CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '') +CI_PAGES_URL = os.getenv('CI_PAGES_URL', '') +CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '') +CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '') diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index b40d827448..5565917940 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -164,7 +164,7 @@ class TestCase: 'name': node.attrib['name'], 'file': node.attrib.get('file'), 'time': float(node.attrib.get('time') or 0), - 'ci_job_url': node.attrib.get('ci_job_url') or '', + 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', } diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index be8fce0af3..b36c540978 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -19,12 +19,16 @@ from prettytable import PrettyTable from .constants import COMMENT_START_MARKER from .constants import REPORT_TEMPLATE_FILEPATH +from .constants import RETRY_JOB_PICTURE_LINK +from .constants import RETRY_JOB_PICTURE_PATH +from .constants import RETRY_JOB_TITLE from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME from .models import GitlabJob from .models import TestCase from .utils import fetch_failed_testcases_failure_ratio from .utils import format_permalink -from .utils import get_report_url +from .utils import get_artifacts_url +from .utils import get_repository_file_url from .utils import is_url from .utils import load_known_failure_cases @@ -69,13 +73,14 @@ class ReportGenerator: # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - report_url: str = get_report_url(job_id, output_filepath) + report_url: str = get_artifacts_url(job_id, output_filepath) return report_url def generate_html_report(self, table_str: str) -> str: # we're using bootstrap table table_str = table_str.replace( - '', '
' + '
', + '
', ) with open(REPORT_TEMPLATE_FILEPATH) as fr: template = fr.read() @@ -245,20 +250,23 @@ class ReportGenerator: if self.mr is None: print('No MR found, skip posting comment') return - + retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n' + f'{RETRY_JOB_PICTURE_LINK}').format(pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH)) + del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})') for note in self.mr.notes.list(iterator=True): if note.body.startswith(COMMENT_START_MARKER): updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body) if updated_str == note.body: # not updated updated_str = f'{note.body.strip()}\n\n{comment}' - note.body = updated_str + updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL) + note.body = updated_str + retry_job_picture_comment note.save() break else: new_comment = f"""{COMMENT_START_MARKER} -{comment}""" +{comment}{retry_job_picture_comment}""" self.mr.notes.create({'body': new_comment}) @@ -526,7 +534,7 @@ class TargetTestReportGenerator(ReportGenerator): 'Test Case', 'Test Script File Path', 'Failure Reason', - 'Failures across all other branches (40 latest testcases)', + 'Cases that failed in other branches as well (40 latest testcases)', 'Dut Log URL', 'Job URL', 'Grafana URL', @@ -534,7 +542,7 @@ class TargetTestReportGenerator(ReportGenerator): row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], value_functions=[ ( - 'Failures across all other branches (40 latest testcases)', + 'Cases that failed in other branches as well (40 latest testcases)', lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", ) ], @@ -696,11 +704,10 @@ class JobReportGenerator(ReportGenerator): ) ], ) - relevant_failed_jobs_report_url = get_report_url(self.job_id, self.failed_jobs_report_file) + relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file) self.additional_info += self.generate_additional_info_section( self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url ) report_str = self.generate_html_report(''.join(report_sections)) - return report_str diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html index bb35367f94..cefa904b46 100644 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -1,88 +1,132 @@ - + - - - {{title}} - - - - - - - -
{{table}}
- - - - - - - - + th:nth-child(1), + td:nth-child(1) { + width: 5%; + } + th:nth-child(2), + td:nth-child(2), + th:nth-child(3), + td:nth-child(3) { + width: 30%; + } + th, + td { + overflow: hidden; + text-overflow: ellipsis; + } + h2 { + margin-top: 10px; + } + .copy-link-icon { + font-size: 20px; + margin-left: 10px; + color: #8f8f97; + cursor: pointer; + } + .copy-link-icon:hover { + color: #282b2c; + } + + + +
{{table}}
+ + + + + + + diff --git a/tools/ci/dynamic_pipelines/templates/retry-jobs.png b/tools/ci/dynamic_pipelines/templates/retry-jobs.png new file mode 100644 index 0000000000..a8a60112c0 Binary files /dev/null and b/tools/ci/dynamic_pipelines/templates/retry-jobs.png differ diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html index 3c75ce390f..3c35a534fd 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html @@ -1,38 +1,61 @@ - + - - - Job Report - - - - - - - -

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

+ + + Job Report + + + + + + + +

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

@@ -70,57 +93,78 @@
Job Name
- - - - - + + + + + - - - + + function scrollToHashLocation() { + const hash = window.location.hash; + if (hash) { + setTimeout(() => { + $("html, body").animate( + { scrollTop: $(hash).offset().top }, + 100 + ); + }, 100); + } + } + + function copyPermalink(anchorId) { + const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`; + history.pushState(null, null, anchorId); + navigator.clipboard.writeText(fullUrl); + scrollToHashLocation(); + } + + function toggleText(e) { + e.preventDefault(); + const link = $(this), + textSpan = link.siblings(".full-text"), + toggleSpan = link.siblings(".text-toggle"); + const visible = textSpan.is(":visible"); + link.text(visible ? "Show More" : "Show Less"); + textSpan.toggle(); + toggleSpan.toggle(); + } + + function setupTextToggles() { + $("table.table td").each(function () { + var cell = $(this); + if (cell.text().length > 100) { + var originalText = cell.text(); + var displayText = + originalText.substring(0, 100) + "..."; + cell.html( + `${displayText}Show More` + ); + } + }); + } + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html index c2d0b95c97..dc63e26152 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html @@ -1,44 +1,67 @@ - + - - - Test Report - - - - - - - -

Failed Test Cases on Other branches (Excludes Known Failure Cases)

+ + + Test Report + + + + + + + +

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- + @@ -51,7 +74,7 @@ - + @@ -60,7 +83,7 @@ - + @@ -69,7 +92,7 @@ - + @@ -78,7 +101,7 @@ - + @@ -87,7 +110,7 @@ - + @@ -96,7 +119,7 @@ - + @@ -105,12 +128,12 @@ - +
Test Case Test Script File Path Failure ReasonFailures across all other branches (40 latest testcases)Cases that failed in other branches as well (40 latest testcases) Dut Log URL Job URL Grafana URLfailed on setup with "EOFError" 0 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt 0 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt 0 / 40 linkNot found link
failed on setup with "EOFError" 3 / 40 linkNot found link
AssertionError: Unity test failed 3 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt 3 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt 3 / 40 linkNot found link

Known Failure Cases

+ onclick="copyPermalink('#known-failure-cases')">
@@ -125,33 +148,33 @@ - + - + - + - +
Test Caseesp32c2.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py AssertionError: Unity test failedNot found link
esp32c3.release.test_esp_timer components/esp_timer/test_apps/pytest_esp_timer_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txtNot found link
esp32c3.512safe.test_wear_levelling components/wear_levelling/test_apps/pytest_wear_levelling.py pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txtNot found link
esp32c3.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txtNot found link

Skipped Test Cases

+ onclick="copyPermalink('#skipped-test-cases')">
@@ -169,7 +192,7 @@
Test Case

Succeeded Test Cases

+ onclick="copyPermalink('#succeeded-test-cases')">
@@ -182,110 +205,131 @@ - + - + - + - + - + - + - + - + - +
Test Case
esp32c2.default.test_vfs_default components/vfs/test_apps/pytest_vfs.pyNot found link
esp32c2.iram.test_vfs_default components/vfs/test_apps/pytest_vfs.pyNot found link
test_python_interpreter_unix test_common.pyNot found link
test_invoke_confserver test_common.pyNot found link
test_ccache_used_to_build test_common.pyNot found link
test_toolchain_prefix_in_description_file test_common.pyNot found link
test_subcommands_with_options test_common.pyNot found link
test_fallback_to_build_system_target test_common.pyNot found link
test_create_component_project test_common.pyNot found link
- - - - - + + + + + - - - + + function scrollToHashLocation() { + const hash = window.location.hash; + if (hash) { + setTimeout(() => { + $("html, body").animate( + { scrollTop: $(hash).offset().top }, + 100 + ); + }, 100); + } + } + + function copyPermalink(anchorId) { + const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`; + history.pushState(null, null, anchorId); + navigator.clipboard.writeText(fullUrl); + scrollToHashLocation(); + } + + function toggleText(e) { + e.preventDefault(); + const link = $(this), + textSpan = link.siblings(".full-text"), + toggleSpan = link.siblings(".text-toggle"); + const visible = textSpan.is(":visible"); + link.text(visible ? "Show More" : "Show Less"); + textSpan.toggle(); + toggleSpan.toggle(); + } + + function setupTextToggles() { + $("table.table td").each(function () { + var cell = $(this); + if (cell.text().length > 100) { + var originalText = cell.text(); + var displayText = + originalText.substring(0, 100) + "..."; + cell.html( + `${displayText}Show More` + ); + } + }); + } + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py index abdd8a7b64..81bae786ce 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py @@ -44,8 +44,8 @@ class TestReportGeneration(unittest.TestCase): self.mock_project.mergerequests.get.return_value = self.mock_mr self.addCleanup(self.gitlab_patcher.stop) - self.addCleanup(self.env_patcher.stop) self.addCleanup(self.failure_rate_patcher.stop) + self.addCleanup(self.env_patcher.stop) self.addCleanup(self.cleanup_files) def cleanup_files(self) -> None: diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py index 79cd3f44d7..08eceb1f9a 100644 --- a/tools/ci/dynamic_pipelines/utils.py +++ b/tools/ci/dynamic_pipelines/utils.py @@ -10,6 +10,11 @@ from urllib.parse import urlparse import requests import yaml +from .constants import CI_DASHBOARD_API +from .constants import CI_JOB_TOKEN +from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA +from .constants import CI_PAGES_URL +from .constants import CI_PROJECT_URL from .models import GitlabJob from .models import Job from .models import TestCase @@ -95,11 +100,9 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: :param commit_id: The commit ID for which to fetch jobs. :return: A list of jobs if the request is successful, otherwise an empty list. """ - token = os.getenv('ESPCI_TOKEN', '') - ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') response = requests.get( - f'{ci_dash_api_backend_host}/commits/{commit_id}/jobs', - headers={'Authorization': f'Bearer {token}'} + f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'} ) if response.status_code != 200: print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') @@ -113,8 +116,8 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] response = requests.post( - f'{ci_dash_api_backend_host}/jobs/failure_ratio', - headers={'Authorization': f'Bearer {token}'}, + f'{CI_DASHBOARD_API}/jobs/failure_ratio', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'}, json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, ) if response.status_code != 200: @@ -139,12 +142,10 @@ def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], bra :param branches_filter: The filter to filter testcases by branch names. :return: A list of testcases with enriched with failure rates data. """ - token = os.getenv('ESPCI_TOKEN', '') - ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter} response = requests.post( - f'{ci_dash_api_backend_host}/testcases/failure_ratio', - headers={'Authorization': f'Bearer {token}'}, + f'{CI_DASHBOARD_API}/testcases/failure_ratio', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'}, json=req_json, ) if response.status_code != 200: @@ -191,13 +192,23 @@ def format_permalink(s: str) -> str: return formatted_string -def get_report_url(job_id: int, output_filepath: str) -> str: +def get_artifacts_url(job_id: int, output_filepath: str) -> str: """ - Generates the url of the path where the report will be stored in the job's artifacts . + Generates the url of the path where the artifact will be stored in the job's artifacts . :param job_id: The job identifier used to construct the URL. :param output_filepath: The path to the output file. :return: The modified URL pointing to the job's artifacts. """ - url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf') return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}' + + +def get_repository_file_url(file_path: str) -> str: + """ + Generates the url of the file path inside the repository. + + :param file_path: The file path where the file is stored. + :return: The modified URL pointing to the file's path in the repository. + """ + return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 1bffbef372..7c83d6f205 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -254,7 +254,7 @@ class Gitlab(object): @staticmethod def decompress_archive(path: str, destination: str) -> str: full_destination = os.path.abspath(destination) - # By default max path lenght is set to 260 characters + # By default max path length is set to 260 characters # Prefix `\\?\` extends it to 32,767 characters if sys.platform == 'win32': full_destination = '\\\\?\\' + full_destination @@ -279,6 +279,29 @@ class Gitlab(object): job = self.project.jobs.get(job_id) return ','.join(job.tag_list) + def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: + """ + Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'. + + :param pipeline_id: ID of the pipeline whose failed jobs are to be retried. + :param retry_allowed_failures: Whether to retry jobs that are marked as allowed failures. + """ + pipeline = self.project.pipelines.get(pipeline_id) + jobs_to_retry = [ + job + for job in pipeline.jobs.list(scope='failed') + if retry_allowed_failures or not job.attributes.get('allow_failure', False) + ] + jobs_succeeded_retry = [] + for job in jobs_to_retry: + try: + res = self.project.jobs.get(job.id).retry() + jobs_succeeded_retry.append(job.id) + logging.info(f'Retried job {job.id} with result {res}') + except Exception as e: + logging.error(f'Failed to retry job {job.id}: {str(e)}') + return jobs_succeeded_retry + def main() -> None: parser = argparse.ArgumentParser() @@ -291,6 +314,9 @@ def main() -> None: parser.add_argument('--project_name', '-m', default=None) parser.add_argument('--destination', '-d', default=None) parser.add_argument('--artifact_path', '-a', nargs='*', default=None) + parser.add_argument( + '--retry-allowed-failures', action='store_true', help='Flag to retry jobs marked as allowed failures' + ) args = parser.parse_args() gitlab_inst = Gitlab(args.project_id) @@ -306,6 +332,9 @@ def main() -> None: elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) print('project id: {}'.format(ret)) + elif args.action == 'retry_failed_jobs': + res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures) + print('job retried successfully: {}'.format(res)) elif args.action == 'get_job_tags': ret = gitlab_inst.get_job_tags(args.job_id) print(ret)