Merge branch 'ci/add-retry-job-func-to-report' into 'master'

ci: add retry job functionality to dynamic pipeline report

Closes IDFCI-2211

See merge request espressif/esp-idf!32147
This commit is contained in:
Aleksei Apaseev 2024-08-12 09:46:02 +08:00
commit 9862ae280f
13 changed files with 512 additions and 307 deletions

View File

@ -31,4 +31,5 @@ include:
- '.gitlab/ci/host-test.yml'
- '.gitlab/ci/deploy.yml'
- '.gitlab/ci/post_deploy.yml'
- '.gitlab/ci/retry_failed_jobs.yml'
- '.gitlab/ci/test-win.yml'

View File

@ -12,6 +12,7 @@ stages:
- test_deploy
- deploy
- post_deploy
- retry_failed_jobs
variables:
# System environment

View File

@ -0,0 +1,14 @@
retry_failed_jobs:
stage: retry_failed_jobs
tags: [shiny, fast_run]
image: $ESP_ENV_IMAGE
dependencies: null
before_script: []
cache: []
extends: []
script:
- echo "Retrieving and retrying all failed jobs for the pipeline..."
- python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
when: manual
needs:
- generate_failed_jobs_report

View File

@ -29,8 +29,18 @@ REPORT_TEMPLATE_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html'
)
RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png'
RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:'
RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})'
BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests'
KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
)
CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '')
CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '')
CI_PAGES_URL = os.getenv('CI_PAGES_URL', '')
CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '')
CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '')

View File

@ -164,7 +164,7 @@ class TestCase:
'name': node.attrib['name'],
'file': node.attrib.get('file'),
'time': float(node.attrib.get('time') or 0),
'ci_job_url': node.attrib.get('ci_job_url') or '',
'ci_job_url': node.attrib.get('ci_job_url') or 'Not found',
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
'dut_log_url': node.attrib.get('dut_log_url') or 'Not found',
}

View File

@ -19,12 +19,16 @@ from prettytable import PrettyTable
from .constants import COMMENT_START_MARKER
from .constants import REPORT_TEMPLATE_FILEPATH
from .constants import RETRY_JOB_PICTURE_LINK
from .constants import RETRY_JOB_PICTURE_PATH
from .constants import RETRY_JOB_TITLE
from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME
from .models import GitlabJob
from .models import TestCase
from .utils import fetch_failed_testcases_failure_ratio
from .utils import format_permalink
from .utils import get_report_url
from .utils import get_artifacts_url
from .utils import get_repository_file_url
from .utils import is_url
from .utils import load_known_failure_cases
@ -69,13 +73,14 @@ class ReportGenerator:
# for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt
# CI_PAGES_URL is {URL}/esp-idf, which missed one `-`
report_url: str = get_report_url(job_id, output_filepath)
report_url: str = get_artifacts_url(job_id, output_filepath)
return report_url
def generate_html_report(self, table_str: str) -> str:
# we're using bootstrap table
table_str = table_str.replace(
'<table>', '<table data-toggle="table" data-search="true" data-sticky-header="true">'
'<table>',
'<table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">',
)
with open(REPORT_TEMPLATE_FILEPATH) as fr:
template = fr.read()
@ -245,20 +250,23 @@ class ReportGenerator:
if self.mr is None:
print('No MR found, skip posting comment')
return
retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n'
f'{RETRY_JOB_PICTURE_LINK}').format(pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH))
del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})')
for note in self.mr.notes.list(iterator=True):
if note.body.startswith(COMMENT_START_MARKER):
updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body)
if updated_str == note.body: # not updated
updated_str = f'{note.body.strip()}\n\n{comment}'
note.body = updated_str
updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL)
note.body = updated_str + retry_job_picture_comment
note.save()
break
else:
new_comment = f"""{COMMENT_START_MARKER}
{comment}"""
{comment}{retry_job_picture_comment}"""
self.mr.notes.create({'body': new_comment})
@ -526,7 +534,7 @@ class TargetTestReportGenerator(ReportGenerator):
'Test Case',
'Test Script File Path',
'Failure Reason',
'Failures across all other branches (40 latest testcases)',
'Cases that failed in other branches as well (40 latest testcases)',
'Dut Log URL',
'Job URL',
'Grafana URL',
@ -534,7 +542,7 @@ class TargetTestReportGenerator(ReportGenerator):
row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
value_functions=[
(
'Failures across all other branches (40 latest testcases)',
'Cases that failed in other branches as well (40 latest testcases)',
lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}",
)
],
@ -696,11 +704,10 @@ class JobReportGenerator(ReportGenerator):
)
],
)
relevant_failed_jobs_report_url = get_report_url(self.job_id, self.failed_jobs_report_file)
relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url
)
report_str = self.generate_html_report(''.join(report_sections))
return report_str

View File

@ -1,88 +1,132 @@
<!doctype html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>{{title}}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" />
<link href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css" rel="stylesheet" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css" rel="stylesheet">
<style>
.text-toggle, .full-text { cursor: pointer; }
th:nth-child(1), td:nth-child(1) { width: 5%; }
th:nth-child(2), td:nth-child(2),
th:nth-child(3), td:nth-child(3) { width: 30%; }
th, td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid">{{table}}</div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on('load', function() {
var hash = window.location.hash;
if (hash) {
setTimeout(function() {
$('html, body').animate({ scrollTop: $(hash).offset().top }, 100);
}, 100);
<head>
<meta charset="utf-8" />
<title>{{title}}</title>
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<link
href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css"
rel="stylesheet"
/>
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
/>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
rel="stylesheet"
/>
<style>
.text-toggle,
.full-text {
cursor: pointer;
}
});
</script>
<script>
function copyPermalink(anchorId) {
const fullUrl = window.location.origin + window.location.pathname + anchorId;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl)
setTimeout(function() {
$('html, body').animate({ scrollTop: $(anchorId).offset().top }, 100);
}, 100);
}
</script>
<script>
$(document).ready(function() {
$('table.table td').each(function() {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText = originalText.substring(0, 100) + '...';
cell.html('<span class="text-toggle">' + displayText + '</span><span class="full-text" style="display: none;">' + originalText + '</span>');
cell.append('<a href="#" class="toggle-link">Show More</a>');
}
});
$('body').on('click', '.toggle-link', function(e) {
e.preventDefault();
var link = $(this);
var textSpan = link.siblings('.full-text');
var toggleSpan = link.siblings('.text-toggle');
if (textSpan.is(':visible')) {
link.text('Show More');
textSpan.hide();
toggleSpan.show();
} else {
link.text('Show Less');
textSpan.show();
toggleSpan.hide();
}
});
});
</script>
</body>
th:nth-child(1),
td:nth-child(1) {
width: 5%;
}
th:nth-child(2),
td:nth-child(2),
th:nth-child(3),
td:nth-child(3) {
width: 30%;
}
th,
td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid">{{table}}</div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on("load", function () {
var hash = window.location.hash;
if (hash) {
setTimeout(function () {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
});
</script>
<script>
$(document).ready(function () {
scrollToHashLocation();
setupTextToggles();
setupEventHandlers();
});
function setupEventHandlers() {
$(window).on("load", scrollToHashLocation);
$("body").on("click", ".toggle-link", toggleText);
}
function scrollToHashLocation() {
const hash = window.location.hash;
if (hash) {
setTimeout(() => {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
}
function copyPermalink(anchorId) {
const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl);
scrollToHashLocation();
}
function toggleText(e) {
e.preventDefault();
const link = $(this),
textSpan = link.siblings(".full-text"),
toggleSpan = link.siblings(".text-toggle");
const visible = textSpan.is(":visible");
link.text(visible ? "Show More" : "Show Less");
textSpan.toggle();
toggleSpan.toggle();
}
function setupTextToggles() {
$("table.table td").each(function () {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText =
originalText.substring(0, 100) + "...";
cell.html(
`<span class="text-toggle">${displayText}</span><span class="full-text" style="display: none;">${originalText}</span><a href="#" class="toggle-link">Show More</a>`
);
}
});
}
</script>
</body>
</html>

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

View File

@ -1,38 +1,61 @@
<!doctype html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Job Report</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" />
<link href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css" rel="stylesheet" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css" rel="stylesheet">
<style>
.text-toggle, .full-text { cursor: pointer; }
th:nth-child(1), td:nth-child(1) { width: 5%; }
th:nth-child(2), td:nth-child(2),
th:nth-child(3), td:nth-child(3) { width: 30%; }
th, td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid"><h2 id="failed-jobs">Failed Jobs (Excludes "integration_test" and "target_test" jobs)<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#failed-jobs')"></i></h2><table data-toggle="table" data-search="true" data-sticky-header="true">
<head>
<meta charset="utf-8" />
<title>Job Report</title>
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<link
href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css"
rel="stylesheet"
/>
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
/>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
rel="stylesheet"
/>
<style>
.text-toggle,
.full-text {
cursor: pointer;
}
th:nth-child(1),
td:nth-child(1) {
width: 5%;
}
th:nth-child(2),
td:nth-child(2),
th:nth-child(3),
td:nth-child(3) {
width: 30%;
}
th,
td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid"><h2 id="failed-jobs">Failed Jobs (Excludes "integration_test" and "target_test" jobs)<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#failed-jobs')"></i></h2><table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">
<thead>
<tr>
<th>Job Name</th>
@ -70,57 +93,78 @@
</tr>
</tbody>
</table></div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on('load', function() {
var hash = window.location.hash;
if (hash) {
setTimeout(function() {
$('html, body').animate({ scrollTop: $(hash).offset().top }, 100);
}, 100);
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on("load", function () {
var hash = window.location.hash;
if (hash) {
setTimeout(function () {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
});
</script>
<script>
$(document).ready(function () {
scrollToHashLocation();
setupTextToggles();
setupEventHandlers();
});
function setupEventHandlers() {
$(window).on("load", scrollToHashLocation);
$("body").on("click", ".toggle-link", toggleText);
}
});
</script>
<script>
function copyPermalink(anchorId) {
const fullUrl = window.location.origin + window.location.pathname + anchorId;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl)
setTimeout(function() {
$('html, body').animate({ scrollTop: $(anchorId).offset().top }, 100);
}, 100);
}
</script>
<script>
$(document).ready(function() {
$('table.table td').each(function() {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText = originalText.substring(0, 100) + '...';
cell.html('<span class="text-toggle">' + displayText + '</span><span class="full-text" style="display: none;">' + originalText + '</span>');
cell.append('<a href="#" class="toggle-link">Show More</a>');
}
});
$('body').on('click', '.toggle-link', function(e) {
e.preventDefault();
var link = $(this);
var textSpan = link.siblings('.full-text');
var toggleSpan = link.siblings('.text-toggle');
if (textSpan.is(':visible')) {
link.text('Show More');
textSpan.hide();
toggleSpan.show();
} else {
link.text('Show Less');
textSpan.show();
toggleSpan.hide();
}
});
});
</script>
</body>
function scrollToHashLocation() {
const hash = window.location.hash;
if (hash) {
setTimeout(() => {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
}
function copyPermalink(anchorId) {
const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl);
scrollToHashLocation();
}
function toggleText(e) {
e.preventDefault();
const link = $(this),
textSpan = link.siblings(".full-text"),
toggleSpan = link.siblings(".text-toggle");
const visible = textSpan.is(":visible");
link.text(visible ? "Show More" : "Show Less");
textSpan.toggle();
toggleSpan.toggle();
}
function setupTextToggles() {
$("table.table td").each(function () {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText =
originalText.substring(0, 100) + "...";
cell.html(
`<span class="text-toggle">${displayText}</span><span class="full-text" style="display: none;">${originalText}</span><a href="#" class="toggle-link">Show More</a>`
);
}
});
}
</script>
</body>
</html>

View File

@ -1,44 +1,67 @@
<!doctype html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Test Report</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" />
<link href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css" rel="stylesheet" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css" rel="stylesheet">
<style>
.text-toggle, .full-text { cursor: pointer; }
th:nth-child(1), td:nth-child(1) { width: 5%; }
th:nth-child(2), td:nth-child(2),
th:nth-child(3), td:nth-child(3) { width: 30%; }
th, td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid"><h2 id="failed-test-cases-on-other-branches">Failed Test Cases on Other branches (Excludes Known Failure Cases)<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#failed-test-cases-on-other-branches')"></i></h2><table data-toggle="table" data-search="true" data-sticky-header="true">
<head>
<meta charset="utf-8" />
<title>Test Report</title>
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<link
href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css"
rel="stylesheet"
/>
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
/>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
rel="stylesheet"
/>
<style>
.text-toggle,
.full-text {
cursor: pointer;
}
th:nth-child(1),
td:nth-child(1) {
width: 5%;
}
th:nth-child(2),
td:nth-child(2),
th:nth-child(3),
td:nth-child(3) {
width: 30%;
}
th,
td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid"><h2 id="failed-test-cases-on-other-branches">Failed Test Cases on Other branches (Excludes Known Failure Cases)<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#failed-test-cases-on-other-branches')"></i></h2><table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">
<thead>
<tr>
<th>Test Case</th>
<th>Test Script File Path</th>
<th>Failure Reason</th>
<th>Failures across all other branches (40 latest testcases)</th>
<th>Cases that failed in other branches as well (40 latest testcases)</th>
<th>Dut Log URL</th>
<th>Job URL</th>
<th>Grafana URL</th>
@ -51,7 +74,7 @@
<td>failed on setup with "EOFError"</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_device/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27defaults%27%2C%20%27defaults%27%29.test_i2c_multi_device">link</a></td>
</tr>
<tr>
@ -60,7 +83,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_esp_timer/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_esp_timer">link</a></td>
</tr>
<tr>
@ -69,7 +92,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td>
<td>0 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.default.test_wpa_supplicant_ut">link</a></td>
</tr>
<tr>
@ -78,7 +101,7 @@
<td>failed on setup with "EOFError"</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_dev/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=%28%27esp32h2%27%2C%20%27esp32h2%27%29.%28%27default%27%2C%20%27default%27%29.test_i2s_multi_dev">link</a></td>
</tr>
<tr>
@ -87,7 +110,7 @@
<td>AssertionError: Unity test failed</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c2.default.test_wpa_supplicant_ut/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_wpa_supplicant_ut">link</a></td>
</tr>
<tr>
@ -96,7 +119,7 @@
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.512safe.test_wear_levelling/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.512safe.test_wear_levelling">link</a></td>
</tr>
<tr>
@ -105,12 +128,12 @@
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt</td>
<td>3 / 40</td>
<td><a href="https://url/esp/esp-idf/pytest-embedded/2024-07-01_10-53-05-207900/esp32c3.release.test_wear_levelling/dut.txt">link</a></td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_wear_levelling">link</a></td>
</tr>
</tbody>
</table><h2 id="known-failure-cases">Known Failure Cases<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#known-failure-cases')"></i></h2><table data-toggle="table" data-search="true" data-sticky-header="true">
onclick="copyPermalink('#known-failure-cases')"></i></h2><table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">
<thead>
<tr>
<th>Test Case</th>
@ -125,33 +148,33 @@
<td>esp32c2.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>AssertionError: Unity test failed</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_wpa_supplicant_ut">link</a></td>
</tr>
<tr>
<td>esp32c3.release.test_esp_timer</td>
<td>components/esp_timer/test_apps/pytest_esp_timer_ut.py</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.release.test_esp_timer">link</a></td>
</tr>
<tr>
<td>esp32c3.512safe.test_wear_levelling</td>
<td>components/wear_levelling/test_apps/pytest_wear_levelling.py</td>
<td>pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.512safe.test_wear_levelling">link</a></td>
</tr>
<tr>
<td>esp32c3.default.test_wpa_supplicant_ut</td>
<td>components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py</td>
<td>pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c3.default.test_wpa_supplicant_ut">link</a></td>
</tr>
</tbody>
</table><h2 id="skipped-test-cases">Skipped Test Cases<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#skipped-test-cases')"></i></h2><table data-toggle="table" data-search="true" data-sticky-header="true">
onclick="copyPermalink('#skipped-test-cases')"></i></h2><table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">
<thead>
<tr>
<th>Test Case</th>
@ -169,7 +192,7 @@
</tr>
</tbody>
</table><h2 id="succeeded-test-cases">Succeeded Test Cases<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#succeeded-test-cases')"></i></h2><table data-toggle="table" data-search="true" data-sticky-header="true">
onclick="copyPermalink('#succeeded-test-cases')"></i></h2><table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">
<thead>
<tr>
<th>Test Case</th>
@ -182,110 +205,131 @@
<tr>
<td>esp32c2.default.test_vfs_default</td>
<td>components/vfs/test_apps/pytest_vfs.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.default.test_vfs_default">link</a></td>
</tr>
<tr>
<td>esp32c2.iram.test_vfs_default</td>
<td>components/vfs/test_apps/pytest_vfs.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=esp32c2.iram.test_vfs_default">link</a></td>
</tr>
<tr>
<td>test_python_interpreter_unix</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_python_interpreter_unix">link</a></td>
</tr>
<tr>
<td>test_invoke_confserver</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_invoke_confserver">link</a></td>
</tr>
<tr>
<td>test_ccache_used_to_build</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_ccache_used_to_build">link</a></td>
</tr>
<tr>
<td>test_toolchain_prefix_in_description_file</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_toolchain_prefix_in_description_file">link</a></td>
</tr>
<tr>
<td>test_subcommands_with_options</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_subcommands_with_options">link</a></td>
</tr>
<tr>
<td>test_fallback_to_build_system_target</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_fallback_to_build_system_target">link</a></td>
</tr>
<tr>
<td>test_create_component_project</td>
<td>test_common.py</td>
<td></td>
<td>Not found</td>
<td><a href="https://test_dashboard_host/d/Ucg477Fnz/case-list?var-case_id=test_create_component_project">link</a></td>
</tr>
</tbody>
</table></div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on('load', function() {
var hash = window.location.hash;
if (hash) {
setTimeout(function() {
$('html, body').animate({ scrollTop: $(hash).offset().top }, 100);
}, 100);
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on("load", function () {
var hash = window.location.hash;
if (hash) {
setTimeout(function () {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
});
</script>
<script>
$(document).ready(function () {
scrollToHashLocation();
setupTextToggles();
setupEventHandlers();
});
function setupEventHandlers() {
$(window).on("load", scrollToHashLocation);
$("body").on("click", ".toggle-link", toggleText);
}
});
</script>
<script>
function copyPermalink(anchorId) {
const fullUrl = window.location.origin + window.location.pathname + anchorId;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl)
setTimeout(function() {
$('html, body').animate({ scrollTop: $(anchorId).offset().top }, 100);
}, 100);
}
</script>
<script>
$(document).ready(function() {
$('table.table td').each(function() {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText = originalText.substring(0, 100) + '...';
cell.html('<span class="text-toggle">' + displayText + '</span><span class="full-text" style="display: none;">' + originalText + '</span>');
cell.append('<a href="#" class="toggle-link">Show More</a>');
}
});
$('body').on('click', '.toggle-link', function(e) {
e.preventDefault();
var link = $(this);
var textSpan = link.siblings('.full-text');
var toggleSpan = link.siblings('.text-toggle');
if (textSpan.is(':visible')) {
link.text('Show More');
textSpan.hide();
toggleSpan.show();
} else {
link.text('Show Less');
textSpan.show();
toggleSpan.hide();
}
});
});
</script>
</body>
function scrollToHashLocation() {
const hash = window.location.hash;
if (hash) {
setTimeout(() => {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
}
function copyPermalink(anchorId) {
const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl);
scrollToHashLocation();
}
function toggleText(e) {
e.preventDefault();
const link = $(this),
textSpan = link.siblings(".full-text"),
toggleSpan = link.siblings(".text-toggle");
const visible = textSpan.is(":visible");
link.text(visible ? "Show More" : "Show Less");
textSpan.toggle();
toggleSpan.toggle();
}
function setupTextToggles() {
$("table.table td").each(function () {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText =
originalText.substring(0, 100) + "...";
cell.html(
`<span class="text-toggle">${displayText}</span><span class="full-text" style="display: none;">${originalText}</span><a href="#" class="toggle-link">Show More</a>`
);
}
});
}
</script>
</body>
</html>

View File

@ -44,8 +44,8 @@ class TestReportGeneration(unittest.TestCase):
self.mock_project.mergerequests.get.return_value = self.mock_mr
self.addCleanup(self.gitlab_patcher.stop)
self.addCleanup(self.env_patcher.stop)
self.addCleanup(self.failure_rate_patcher.stop)
self.addCleanup(self.env_patcher.stop)
self.addCleanup(self.cleanup_files)
def cleanup_files(self) -> None:

View File

@ -10,6 +10,11 @@ from urllib.parse import urlparse
import requests
import yaml
from .constants import CI_DASHBOARD_API
from .constants import CI_JOB_TOKEN
from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
from .constants import CI_PAGES_URL
from .constants import CI_PROJECT_URL
from .models import GitlabJob
from .models import Job
from .models import TestCase
@ -95,11 +100,9 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]:
:param commit_id: The commit ID for which to fetch jobs.
:return: A list of jobs if the request is successful, otherwise an empty list.
"""
token = os.getenv('ESPCI_TOKEN', '')
ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '')
response = requests.get(
f'{ci_dash_api_backend_host}/commits/{commit_id}/jobs',
headers={'Authorization': f'Bearer {token}'}
f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs',
headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'}
)
if response.status_code != 200:
print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}')
@ -113,8 +116,8 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]:
failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed']
response = requests.post(
f'{ci_dash_api_backend_host}/jobs/failure_ratio',
headers={'Authorization': f'Bearer {token}'},
f'{CI_DASHBOARD_API}/jobs/failure_ratio',
headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'},
json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]},
)
if response.status_code != 200:
@ -139,12 +142,10 @@ def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], bra
:param branches_filter: The filter to filter testcases by branch names.
:return: A list of testcases with enriched with failure rates data.
"""
token = os.getenv('ESPCI_TOKEN', '')
ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '')
req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter}
response = requests.post(
f'{ci_dash_api_backend_host}/testcases/failure_ratio',
headers={'Authorization': f'Bearer {token}'},
f'{CI_DASHBOARD_API}/testcases/failure_ratio',
headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'},
json=req_json,
)
if response.status_code != 200:
@ -191,13 +192,23 @@ def format_permalink(s: str) -> str:
return formatted_string
def get_report_url(job_id: int, output_filepath: str) -> str:
def get_artifacts_url(job_id: int, output_filepath: str) -> str:
"""
Generates the url of the path where the report will be stored in the job's artifacts .
Generates the url of the path where the artifact will be stored in the job's artifacts .
:param job_id: The job identifier used to construct the URL.
:param output_filepath: The path to the output file.
:return: The modified URL pointing to the job's artifacts.
"""
url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf')
url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf')
return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}'
def get_repository_file_url(file_path: str) -> str:
"""
Generates the url of the file path inside the repository.
:param file_path: The file path where the file is stored.
:return: The modified URL pointing to the file's path in the repository.
"""
return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}'

View File

@ -254,7 +254,7 @@ class Gitlab(object):
@staticmethod
def decompress_archive(path: str, destination: str) -> str:
full_destination = os.path.abspath(destination)
# By default max path lenght is set to 260 characters
# By default max path length is set to 260 characters
# Prefix `\\?\` extends it to 32,767 characters
if sys.platform == 'win32':
full_destination = '\\\\?\\' + full_destination
@ -279,6 +279,29 @@ class Gitlab(object):
job = self.project.jobs.get(job_id)
return ','.join(job.tag_list)
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
"""
Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'.
:param pipeline_id: ID of the pipeline whose failed jobs are to be retried.
:param retry_allowed_failures: Whether to retry jobs that are marked as allowed failures.
"""
pipeline = self.project.pipelines.get(pipeline_id)
jobs_to_retry = [
job
for job in pipeline.jobs.list(scope='failed')
if retry_allowed_failures or not job.attributes.get('allow_failure', False)
]
jobs_succeeded_retry = []
for job in jobs_to_retry:
try:
res = self.project.jobs.get(job.id).retry()
jobs_succeeded_retry.append(job.id)
logging.info(f'Retried job {job.id} with result {res}')
except Exception as e:
logging.error(f'Failed to retry job {job.id}: {str(e)}')
return jobs_succeeded_retry
def main() -> None:
parser = argparse.ArgumentParser()
@ -291,6 +314,9 @@ def main() -> None:
parser.add_argument('--project_name', '-m', default=None)
parser.add_argument('--destination', '-d', default=None)
parser.add_argument('--artifact_path', '-a', nargs='*', default=None)
parser.add_argument(
'--retry-allowed-failures', action='store_true', help='Flag to retry jobs marked as allowed failures'
)
args = parser.parse_args()
gitlab_inst = Gitlab(args.project_id)
@ -306,6 +332,9 @@ def main() -> None:
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print('project id: {}'.format(ret))
elif args.action == 'retry_failed_jobs':
res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures)
print('job retried successfully: {}'.format(res))
elif args.action == 'get_job_tags':
ret = gitlab_inst.get_job_tags(args.job_id)
print(ret)