Skip to content

Commit

Permalink
feat(ci): add retry job functionality to dynamic pipeline report
Browse files Browse the repository at this point in the history
Introduced changes:

- add a manual ci job to retry failed jobs.
- refactor js scripts in report template
- extract the CI ENV vars related to the report generation script to the predefined constants.py module
- introduce a new action "retry_failed_jobs" in helper script "gitlab_api.py"
  • Loading branch information
alekseiapa committed Aug 9, 2024
1 parent f08926b commit a6b84b5
Show file tree
Hide file tree
Showing 13 changed files with 512 additions and 307 deletions.
1 change: 1 addition & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,5 @@ include:
- '.gitlab/ci/host-test.yml'
- '.gitlab/ci/deploy.yml'
- '.gitlab/ci/post_deploy.yml'
- '.gitlab/ci/retry_failed_jobs.yml'
- '.gitlab/ci/test-win.yml'
1 change: 1 addition & 0 deletions .gitlab/ci/common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ stages:
- test_deploy
- deploy
- post_deploy
- retry_failed_jobs

variables:
# System environment
Expand Down
14 changes: 14 additions & 0 deletions .gitlab/ci/retry_failed_jobs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
retry_failed_jobs:
stage: retry_failed_jobs
tags: [shiny, fast_run]
image: $ESP_ENV_IMAGE
dependencies: null
before_script: []
cache: []
extends: []
script:
- echo "Retrieving and retrying all failed jobs for the pipeline..."
- python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
when: manual
needs:
- generate_failed_jobs_report
10 changes: 10 additions & 0 deletions tools/ci/dynamic_pipelines/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,18 @@
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html'
)

RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png'
RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:'
RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})'

BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests'

KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
)

CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '')
CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '')
CI_PAGES_URL = os.getenv('CI_PAGES_URL', '')
CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '')
CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '')
2 changes: 1 addition & 1 deletion tools/ci/dynamic_pipelines/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']:
'name': node.attrib['name'],
'file': node.attrib.get('file'),
'time': float(node.attrib.get('time') or 0),
'ci_job_url': node.attrib.get('ci_job_url') or '',
'ci_job_url': node.attrib.get('ci_job_url') or 'Not found',
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
'dut_log_url': node.attrib.get('dut_log_url') or 'Not found',
}
Expand Down
27 changes: 17 additions & 10 deletions tools/ci/dynamic_pipelines/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@

from .constants import COMMENT_START_MARKER
from .constants import REPORT_TEMPLATE_FILEPATH
from .constants import RETRY_JOB_PICTURE_LINK
from .constants import RETRY_JOB_PICTURE_PATH
from .constants import RETRY_JOB_TITLE
from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME
from .models import GitlabJob
from .models import TestCase
from .utils import fetch_failed_testcases_failure_ratio
from .utils import format_permalink
from .utils import get_report_url
from .utils import get_artifacts_url
from .utils import get_repository_file_url
from .utils import is_url
from .utils import load_known_failure_cases

Expand Down Expand Up @@ -69,13 +73,14 @@ def write_report_to_file(self, report_str: str, job_id: int, output_filepath: st

# for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt
# CI_PAGES_URL is {URL}/esp-idf, which missed one `-`
report_url: str = get_report_url(job_id, output_filepath)
report_url: str = get_artifacts_url(job_id, output_filepath)
return report_url

def generate_html_report(self, table_str: str) -> str:
# we're using bootstrap table
table_str = table_str.replace(
'<table>', '<table data-toggle="table" data-search="true" data-sticky-header="true">'
'<table>',
'<table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">',
)
with open(REPORT_TEMPLATE_FILEPATH) as fr:
template = fr.read()
Expand Down Expand Up @@ -245,20 +250,23 @@ def post_report(self, print_report_path: bool = True) -> None:
if self.mr is None:
print('No MR found, skip posting comment')
return

retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n'
f'{RETRY_JOB_PICTURE_LINK}').format(pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH))
del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})')
for note in self.mr.notes.list(iterator=True):
if note.body.startswith(COMMENT_START_MARKER):
updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body)
if updated_str == note.body: # not updated
updated_str = f'{note.body.strip()}\n\n{comment}'

note.body = updated_str
updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL)
note.body = updated_str + retry_job_picture_comment
note.save()
break
else:
new_comment = f"""{COMMENT_START_MARKER}
{comment}"""
{comment}{retry_job_picture_comment}"""
self.mr.notes.create({'body': new_comment})


Expand Down Expand Up @@ -526,15 +534,15 @@ def get_failed_cases_report_parts(self) -> t.List[str]:
'Test Case',
'Test Script File Path',
'Failure Reason',
'Failures across all other branches (40 latest testcases)',
'Cases that failed in other branches as well (40 latest testcases)',
'Dut Log URL',
'Job URL',
'Grafana URL',
],
row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'],
value_functions=[
(
'Failures across all other branches (40 latest testcases)',
'Cases that failed in other branches as well (40 latest testcases)',
lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}",
)
],
Expand Down Expand Up @@ -696,11 +704,10 @@ def _get_report_str(self) -> str:
)
],
)
relevant_failed_jobs_report_url = get_report_url(self.job_id, self.failed_jobs_report_file)
relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url
)

report_str = self.generate_html_report(''.join(report_sections))

return report_str
214 changes: 129 additions & 85 deletions tools/ci/dynamic_pipelines/templates/report.template.html
Original file line number Diff line number Diff line change
@@ -1,88 +1,132 @@
<!doctype html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>{{title}}</title>
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" />
<link href="https://unpkg.com/[email protected]/dist/bootstrap-table.min.css" rel="stylesheet" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/sticky-header/bootstrap-table-sticky-header.css">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css" rel="stylesheet">
<style>
.text-toggle, .full-text { cursor: pointer; }
th:nth-child(1), td:nth-child(1) { width: 5%; }
th:nth-child(2), td:nth-child(2),
th:nth-child(3), td:nth-child(3) { width: 30%; }
th, td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid">{{table}}</div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/[email protected]/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on('load', function() {
var hash = window.location.hash;
if (hash) {
setTimeout(function() {
$('html, body').animate({ scrollTop: $(hash).offset().top }, 100);
}, 100);
<head>
<meta charset="utf-8" />
<title>{{title}}</title>
<link
href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<link
href="https://unpkg.com/[email protected]/dist/bootstrap-table.min.css"
rel="stylesheet"
/>
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
/>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
rel="stylesheet"
/>
<style>
.text-toggle,
.full-text {
cursor: pointer;
}
});
</script>
<script>
function copyPermalink(anchorId) {
const fullUrl = window.location.origin + window.location.pathname + anchorId;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl)
setTimeout(function() {
$('html, body').animate({ scrollTop: $(anchorId).offset().top }, 100);
}, 100);
}
</script>
<script>
$(document).ready(function() {
$('table.table td').each(function() {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText = originalText.substring(0, 100) + '...';
cell.html('<span class="text-toggle">' + displayText + '</span><span class="full-text" style="display: none;">' + originalText + '</span>');
cell.append('<a href="#" class="toggle-link">Show More</a>');
}
});
$('body').on('click', '.toggle-link', function(e) {
e.preventDefault();
var link = $(this);
var textSpan = link.siblings('.full-text');
var toggleSpan = link.siblings('.text-toggle');
if (textSpan.is(':visible')) {
link.text('Show More');
textSpan.hide();
toggleSpan.show();
} else {
link.text('Show Less');
textSpan.show();
toggleSpan.hide();
}
});
});
</script>
</body>
th:nth-child(1),
td:nth-child(1) {
width: 5%;
}
th:nth-child(2),
td:nth-child(2),
th:nth-child(3),
td:nth-child(3) {
width: 30%;
}
th,
td {
overflow: hidden;
text-overflow: ellipsis;
}
h2 {
margin-top: 10px;
}
.copy-link-icon {
font-size: 20px;
margin-left: 10px;
color: #8f8f97;
cursor: pointer;
}
.copy-link-icon:hover {
color: #282b2c;
}
</style>
</head>
<body>
<div class="container-fluid">{{table}}</div>
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/[email protected]/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script>
$(window).on("load", function () {
var hash = window.location.hash;
if (hash) {
setTimeout(function () {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
});
</script>
<script>
$(document).ready(function () {
scrollToHashLocation();
setupTextToggles();
setupEventHandlers();
});

function setupEventHandlers() {
$(window).on("load", scrollToHashLocation);
$("body").on("click", ".toggle-link", toggleText);
}

function scrollToHashLocation() {
const hash = window.location.hash;
if (hash) {
setTimeout(() => {
$("html, body").animate(
{ scrollTop: $(hash).offset().top },
100
);
}, 100);
}
}

function copyPermalink(anchorId) {
const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`;
history.pushState(null, null, anchorId);
navigator.clipboard.writeText(fullUrl);
scrollToHashLocation();
}

function toggleText(e) {
e.preventDefault();
const link = $(this),
textSpan = link.siblings(".full-text"),
toggleSpan = link.siblings(".text-toggle");
const visible = textSpan.is(":visible");
link.text(visible ? "Show More" : "Show Less");
textSpan.toggle();
toggleSpan.toggle();
}

function setupTextToggles() {
$("table.table td").each(function () {
var cell = $(this);
if (cell.text().length > 100) {
var originalText = cell.text();
var displayText =
originalText.substring(0, 100) + "...";
cell.html(
`<span class="text-toggle">${displayText}</span><span class="full-text" style="display: none;">${originalText}</span><a href="#" class="toggle-link">Show More</a>`
);
}
});
}
</script>
</body>
</html>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit a6b84b5

Please sign in to comment.