From 56f8b7158c71f8a3890c836583699f7573f41771 Mon Sep 17 00:00:00 2001
From: Fu Hanxi <fuhanxi@espressif.com>
Date: Tue, 27 Aug 2024 14:15:16 +0200
Subject: [PATCH] ci: support diff manifest sha with the base commit

---
 .gitignore                                    |  1 +
 .gitlab/ci/build.yml                          |  2 ++
 .gitlab/ci/pre_check.yml                      | 20 +++++++++++++++++++
 .../scripts/generate_build_child_pipeline.py  | 10 ++++++++++
 tools/ci/idf_pytest/script.py                 |  4 ++++
 5 files changed, 37 insertions(+)

diff --git a/.gitignore b/.gitignore
index f70980b703f2..313692055c1a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -102,6 +102,7 @@ pytest_embedded_log/
 list_job*.txt
 size_info*.txt
 XUNIT_RESULT*.xml
+.manifest_sha
 
 # clang config (for LSP)
 .clangd
diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml
index 4255a189a9b2..ce6ccfd2764f 100644
--- a/.gitlab/ci/build.yml
+++ b/.gitlab/ci/build.yml
@@ -300,6 +300,8 @@ generate_build_child_pipeline:
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - pipeline_variables
+    - job: baseline_manifest_sha
+      optional: true
   artifacts:
     paths:
       - build_child_pipeline.yml
diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml
index 3aa4d7807bf5..7bf24fb158fd 100644
--- a/.gitlab/ci/pre_check.yml
+++ b/.gitlab/ci/pre_check.yml
@@ -169,6 +169,26 @@ pipeline_variables:
     expire_in: 1 week
     when: always
 
+baseline_manifest_sha:
+  extends:
+    - .pre_check_template
+    - .rules:dev-push
+  tags: [fast_run, shiny]
+  script:
+    - |
+      if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
+        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
+        git checkout FETCH_HEAD
+        idf-build-apps dump-manifest-sha \
+          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
+          --output .manifest_sha
+      fi
+  artifacts:
+    paths:
+      - .manifest_sha
+    expire_in: 1 week
+    when: always
+
 redundant_pass_job:
   stage: pre_check
   tags: [shiny, fast_run]
diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
index 5c2cf826cc96..b0e9644ee20d 100644
--- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
+++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
@@ -90,6 +90,7 @@ def main(arguments: argparse.Namespace) -> None:
             marker_expr='not host_test',
             config_rules_str=DEFAULT_CONFIG_RULES_STR,
             extra_default_build_targets=extra_default_build_targets,
+            compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath,
             modified_components=arguments.modified_components,
             modified_files=arguments.modified_files,
             ignore_app_dependencies_components=arguments.ignore_app_dependencies_components,
@@ -167,6 +168,11 @@ def main(arguments: argparse.Namespace) -> None:
         default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
         help='default build test rules config file',
     )
+    parser.add_argument(
+        '--compare-manifest-sha-filepath',
+        default=os.path.join(IDF_PATH, '.manifest_sha'),
+        help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`'
+    )
     parser.add_argument(
         '--modified-components',
         type=_separate_str_to_list,
@@ -244,4 +250,8 @@ def main(arguments: argparse.Namespace) -> None:
             # setting default values
             args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS
 
+    if not os.path.isfile(args.compare_manifest_sha_filepath):
+        # ignore if the file does not exist
+        args.compare_manifest_sha_filepath = None
+
     main(args)
diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py
index 199fe7b03f38..be99053ef66d 100644
--- a/tools/ci/idf_pytest/script.py
+++ b/tools/ci/idf_pytest/script.py
@@ -128,6 +128,7 @@ def get_all_apps(
     config_rules_str: t.Optional[t.List[str]] = None,
     preserve_all: bool = False,
     extra_default_build_targets: t.Optional[t.List[str]] = None,
+    compare_manifest_sha_filepath: t.Optional[str] = None,
     modified_components: t.Optional[t.List[str]] = None,
     modified_files: t.Optional[t.List[str]] = None,
     ignore_app_dependencies_components: t.Optional[t.List[str]] = None,
@@ -143,8 +144,10 @@ def get_all_apps(
     :param config_rules_str: config rules string
     :param preserve_all: preserve all apps
     :param extra_default_build_targets: extra default build targets
+    :param compare_manifest_sha_filepath: check manifest sha filepath
     :param modified_components: modified components
     :param modified_files: modified files
+    :param ignore_app_dependencies_components: ignore app dependencies components
     :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns
     :return: tuple of test-required apps and non-test-related apps
     """
@@ -162,6 +165,7 @@ def get_all_apps(
             size_json_filename='size.json',
             check_warnings=True,
             manifest_rootpath=IDF_PATH,
+            compare_manifest_sha_filepath=compare_manifest_sha_filepath,
             manifest_files=get_all_manifest_files(),
             default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []),
             modified_components=modified_components,