diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc0403e0c42..0e449ec7e9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,4 +31,5 @@ include: - '.gitlab/ci/host-test.yml' - '.gitlab/ci/deploy.yml' - '.gitlab/ci/post_deploy.yml' + - '.gitlab/ci/retry_failed_jobs.yml' - '.gitlab/ci/test-win.yml' diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index b43e1d3d570..223a1f9dd9f 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -12,6 +12,7 @@ stages: - test_deploy - deploy - post_deploy + - retry_failed_jobs variables: # System environment diff --git a/.gitlab/ci/retry_failed_jobs.yml b/.gitlab/ci/retry_failed_jobs.yml new file mode 100644 index 00000000000..28a2c1e06f7 --- /dev/null +++ b/.gitlab/ci/retry_failed_jobs.yml @@ -0,0 +1,14 @@ +retry_failed_jobs: + stage: retry_failed_jobs + tags: [shiny, fast_run] + image: $ESP_ENV_IMAGE + dependencies: null + before_script: [] + cache: [] + extends: [] + script: + - echo "Retrieving and retrying all failed jobs for the pipeline..." + - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID + when: manual + needs: + - generate_failed_jobs_report diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 21f903338df..bbeefcfc4df 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -29,8 +29,18 @@ IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' ) +RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png' +RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:' +RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})' + BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) + +CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '') +CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '') +CI_PAGES_URL = os.getenv('CI_PAGES_URL', '') +CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '') +CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '') diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index b40d827448d..55659179401 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -164,7 +164,7 @@ def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: 'name': node.attrib['name'], 'file': node.attrib.get('file'), 'time': float(node.attrib.get('time') or 0), - 'ci_job_url': node.attrib.get('ci_job_url') or '', + 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', } diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index be8fce0af35..b36c5409788 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -19,12 +19,16 @@ from .constants import COMMENT_START_MARKER from .constants import REPORT_TEMPLATE_FILEPATH +from .constants import RETRY_JOB_PICTURE_LINK +from .constants import RETRY_JOB_PICTURE_PATH +from .constants import RETRY_JOB_TITLE from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME from .models import GitlabJob from .models import TestCase from .utils import fetch_failed_testcases_failure_ratio from .utils import format_permalink -from .utils import get_report_url +from .utils import get_artifacts_url +from .utils import get_repository_file_url from .utils import is_url from .utils import load_known_failure_cases @@ -69,13 +73,14 @@ def write_report_to_file(self, report_str: str, job_id: int, output_filepath: st # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - report_url: str = get_report_url(job_id, output_filepath) + report_url: str = get_artifacts_url(job_id, output_filepath) return report_url def generate_html_report(self, table_str: str) -> str: # we're using bootstrap table table_str = table_str.replace( - '', '
' + '
', + '
', ) with open(REPORT_TEMPLATE_FILEPATH) as fr: template = fr.read() @@ -245,20 +250,23 @@ def post_report(self, print_report_path: bool = True) -> None: if self.mr is None: print('No MR found, skip posting comment') return - + retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n' + f'{RETRY_JOB_PICTURE_LINK}').format(pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH)) + del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})') for note in self.mr.notes.list(iterator=True): if note.body.startswith(COMMENT_START_MARKER): updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body) if updated_str == note.body: # not updated updated_str = f'{note.body.strip()}\n\n{comment}' - note.body = updated_str + updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL) + note.body = updated_str + retry_job_picture_comment note.save() break else: new_comment = f"""{COMMENT_START_MARKER} -{comment}""" +{comment}{retry_job_picture_comment}""" self.mr.notes.create({'body': new_comment}) @@ -526,7 +534,7 @@ def get_failed_cases_report_parts(self) -> t.List[str]: 'Test Case', 'Test Script File Path', 'Failure Reason', - 'Failures across all other branches (40 latest testcases)', + 'Cases that failed in other branches as well (40 latest testcases)', 'Dut Log URL', 'Job URL', 'Grafana URL', @@ -534,7 +542,7 @@ def get_failed_cases_report_parts(self) -> t.List[str]: row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], value_functions=[ ( - 'Failures across all other branches (40 latest testcases)', + 'Cases that failed in other branches as well (40 latest testcases)', lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", ) ], @@ -696,11 +704,10 @@ def _get_report_str(self) -> str: ) ], ) - relevant_failed_jobs_report_url = get_report_url(self.job_id, self.failed_jobs_report_file) + relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file) self.additional_info += self.generate_additional_info_section( self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url ) report_str = self.generate_html_report(''.join(report_sections)) - return report_str diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html index bb35367f941..cefa904b462 100644 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -1,88 +1,132 @@ - + - - - {{title}} - - - - - - - -
{{table}}
- - - - - - - - + th:nth-child(1), + td:nth-child(1) { + width: 5%; + } + th:nth-child(2), + td:nth-child(2), + th:nth-child(3), + td:nth-child(3) { + width: 30%; + } + th, + td { + overflow: hidden; + text-overflow: ellipsis; + } + h2 { + margin-top: 10px; + } + .copy-link-icon { + font-size: 20px; + margin-left: 10px; + color: #8f8f97; + cursor: pointer; + } + .copy-link-icon:hover { + color: #282b2c; + } + + + +
{{table}}
+ + + + + + + diff --git a/tools/ci/dynamic_pipelines/templates/retry-jobs.png b/tools/ci/dynamic_pipelines/templates/retry-jobs.png new file mode 100644 index 00000000000..a8a60112c01 Binary files /dev/null and b/tools/ci/dynamic_pipelines/templates/retry-jobs.png differ diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html index 3c75ce390f2..3c35a534fdd 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html @@ -1,38 +1,61 @@ - + - - - Job Report - - - - - - - -

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

+ + + Job Report + + + + + + + +

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

@@ -70,57 +93,78 @@
Job Name
- - - - - + + + + + - - - + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html index c2d0b95c970..dc63e261523 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html @@ -1,44 +1,67 @@ - + - - - Test Report - - - - - - - -

Failed Test Cases on Other branches (Excludes Known Failure Cases)

+ + + Test Report + + + + + + + +

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- + @@ -51,7 +74,7 @@ - + @@ -60,7 +83,7 @@ - + @@ -69,7 +92,7 @@ - + @@ -78,7 +101,7 @@ - + @@ -87,7 +110,7 @@ - + @@ -96,7 +119,7 @@ - + @@ -105,12 +128,12 @@ - +
Test Case Test Script File Path Failure ReasonFailures across all other branches (40 latest testcases)Cases that failed in other branches as well (40 latest testcases) Dut Log URL Job URL Grafana URLfailed on setup with "EOFError" 0 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt 0 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt 0 / 40 linkNot found link
failed on setup with "EOFError" 3 / 40 linkNot found link
AssertionError: Unity test failed 3 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt 3 / 40 linkNot found link
pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt 3 / 40 linkNot found link

Known Failure Cases

+ onclick="copyPermalink('#known-failure-cases')">
@@ -125,33 +148,33 @@ - + - + - + - +
Test Caseesp32c2.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py AssertionError: Unity test failedNot found link
esp32c3.release.test_esp_timer components/esp_timer/test_apps/pytest_esp_timer_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txtNot found link
esp32c3.512safe.test_wear_levelling components/wear_levelling/test_apps/pytest_wear_levelling.py pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txtNot found link
esp32c3.default.test_wpa_supplicant_ut components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txtNot found link

Skipped Test Cases

+ onclick="copyPermalink('#skipped-test-cases')">
@@ -169,7 +192,7 @@
Test Case

Succeeded Test Cases

+ onclick="copyPermalink('#succeeded-test-cases')">
@@ -182,110 +205,131 @@ - + - + - + - + - + - + - + - + - +
Test Case
esp32c2.default.test_vfs_default components/vfs/test_apps/pytest_vfs.pyNot found link
esp32c2.iram.test_vfs_default components/vfs/test_apps/pytest_vfs.pyNot found link
test_python_interpreter_unix test_common.pyNot found link
test_invoke_confserver test_common.pyNot found link
test_ccache_used_to_build test_common.pyNot found link
test_toolchain_prefix_in_description_file test_common.pyNot found link
test_subcommands_with_options test_common.pyNot found link
test_fallback_to_build_system_target test_common.pyNot found link
test_create_component_project test_common.pyNot found link
- - - - - + + + + + - - - + + diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py index abdd8a7b646..81bae786ced 100644 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py +++ b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py @@ -44,8 +44,8 @@ def setup_patches(self) -> None: self.mock_project.mergerequests.get.return_value = self.mock_mr self.addCleanup(self.gitlab_patcher.stop) - self.addCleanup(self.env_patcher.stop) self.addCleanup(self.failure_rate_patcher.stop) + self.addCleanup(self.env_patcher.stop) self.addCleanup(self.cleanup_files) def cleanup_files(self) -> None: diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py index 79cd3f44d71..08eceb1f9a7 100644 --- a/tools/ci/dynamic_pipelines/utils.py +++ b/tools/ci/dynamic_pipelines/utils.py @@ -10,6 +10,11 @@ import requests import yaml +from .constants import CI_DASHBOARD_API +from .constants import CI_JOB_TOKEN +from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA +from .constants import CI_PAGES_URL +from .constants import CI_PROJECT_URL from .models import GitlabJob from .models import Job from .models import TestCase @@ -95,11 +100,9 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: :param commit_id: The commit ID for which to fetch jobs. :return: A list of jobs if the request is successful, otherwise an empty list. """ - token = os.getenv('ESPCI_TOKEN', '') - ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') response = requests.get( - f'{ci_dash_api_backend_host}/commits/{commit_id}/jobs', - headers={'Authorization': f'Bearer {token}'} + f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'} ) if response.status_code != 200: print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') @@ -113,8 +116,8 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] response = requests.post( - f'{ci_dash_api_backend_host}/jobs/failure_ratio', - headers={'Authorization': f'Bearer {token}'}, + f'{CI_DASHBOARD_API}/jobs/failure_ratio', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'}, json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, ) if response.status_code != 200: @@ -139,12 +142,10 @@ def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], bra :param branches_filter: The filter to filter testcases by branch names. :return: A list of testcases with enriched with failure rates data. """ - token = os.getenv('ESPCI_TOKEN', '') - ci_dash_api_backend_host = os.getenv('CI_DASHBOARD_API', '') req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter} response = requests.post( - f'{ci_dash_api_backend_host}/testcases/failure_ratio', - headers={'Authorization': f'Bearer {token}'}, + f'{CI_DASHBOARD_API}/testcases/failure_ratio', + headers={'Authorization': f'Bearer {CI_JOB_TOKEN}'}, json=req_json, ) if response.status_code != 200: @@ -191,13 +192,23 @@ def format_permalink(s: str) -> str: return formatted_string -def get_report_url(job_id: int, output_filepath: str) -> str: +def get_artifacts_url(job_id: int, output_filepath: str) -> str: """ - Generates the url of the path where the report will be stored in the job's artifacts . + Generates the url of the path where the artifact will be stored in the job's artifacts . :param job_id: The job identifier used to construct the URL. :param output_filepath: The path to the output file. :return: The modified URL pointing to the job's artifacts. """ - url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf') return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}' + + +def get_repository_file_url(file_path: str) -> str: + """ + Generates the url of the file path inside the repository. + + :param file_path: The file path where the file is stored. + :return: The modified URL pointing to the file's path in the repository. + """ + return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 1bffbef372f..7c83d6f205a 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -254,7 +254,7 @@ def download_archive(self, ref: str, destination: str, project_id: Optional[int] @staticmethod def decompress_archive(path: str, destination: str) -> str: full_destination = os.path.abspath(destination) - # By default max path lenght is set to 260 characters + # By default max path length is set to 260 characters # Prefix `\\?\` extends it to 32,767 characters if sys.platform == 'win32': full_destination = '\\\\?\\' + full_destination @@ -279,6 +279,29 @@ def get_job_tags(self, job_id: int) -> str: job = self.project.jobs.get(job_id) return ','.join(job.tag_list) + def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: + """ + Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'. + + :param pipeline_id: ID of the pipeline whose failed jobs are to be retried. + :param retry_allowed_failures: Whether to retry jobs that are marked as allowed failures. + """ + pipeline = self.project.pipelines.get(pipeline_id) + jobs_to_retry = [ + job + for job in pipeline.jobs.list(scope='failed') + if retry_allowed_failures or not job.attributes.get('allow_failure', False) + ] + jobs_succeeded_retry = [] + for job in jobs_to_retry: + try: + res = self.project.jobs.get(job.id).retry() + jobs_succeeded_retry.append(job.id) + logging.info(f'Retried job {job.id} with result {res}') + except Exception as e: + logging.error(f'Failed to retry job {job.id}: {str(e)}') + return jobs_succeeded_retry + def main() -> None: parser = argparse.ArgumentParser() @@ -291,6 +314,9 @@ def main() -> None: parser.add_argument('--project_name', '-m', default=None) parser.add_argument('--destination', '-d', default=None) parser.add_argument('--artifact_path', '-a', nargs='*', default=None) + parser.add_argument( + '--retry-allowed-failures', action='store_true', help='Flag to retry jobs marked as allowed failures' + ) args = parser.parse_args() gitlab_inst = Gitlab(args.project_id) @@ -306,6 +332,9 @@ def main() -> None: elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) print('project id: {}'.format(ret)) + elif args.action == 'retry_failed_jobs': + res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures) + print('job retried successfully: {}'.format(res)) elif args.action == 'get_job_tags': ret = gitlab_inst.get_job_tags(args.job_id) print(ret)