fix the error when lucia URL is not available

.github/workflows/build-all.yaml

@@ -100,7 +100,8 @@ jobs:
                 -n \
                 -c $GITHUB_WORKSPACE/config/cluster-configuration \
                 -s alert-manager \
-                -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring
+                -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring
+              continue
             fi
             echo "Building service: $service"
             if python3 $GITHUB_WORKSPACE/build/pai_build.py build \

.github/workflows/build-deploy-changes.yaml

@@ -123,24 +123,43 @@ jobs:
             $GITHUB_WORKSPACE/build/pai_build.py build \
               -n \
               -c $GITHUB_WORKSPACE/config/cluster-configuration \
-              -s alert-manager
-              -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring
+              -s alert-manager \
+              -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring
           fi
 
+          changed_services=$(echo $changed_services | xargs)
           echo "Changed services after removing alert-manager: $changed_services"
-          $GITHUB_WORKSPACE/build/pai_build.py build \
-            -n \
-            -c $GITHUB_WORKSPACE/config/cluster-configuration \
-            -s $changed_services
+          if [ -n "$changed_services" ]; then
+            $GITHUB_WORKSPACE/build/pai_build.py build \
+              -n \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s $changed_services
+          fi
 
       - name: Push Images of Changed Services to ACR
         if: steps.check.outputs.has_changed == 'true'
+        shell: bash
         run: |
           changed_services="${{ steps.changes.outputs.folders }}"
           echo "Pushing: $changed_services"
-          $GITHUB_WORKSPACE/build/pai_build.py push \
-            -c $GITHUB_WORKSPACE/config/cluster-configuration \
-            -s $changed_services
+          if [[ "$changed_services" == *"alert-manager"* ]]; then
+            echo "alert-manager is in the changed services"
+            changed_services=$(echo $changed_services | sed 's/alert-manager//g')
+            # push specific images in alert-manager to ACR
+            echo "Pushing specific alert-manager images to ACR"
+            $GITHUB_WORKSPACE/build/pai_build.py push \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s alert-manager \
+              -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring
+          fi
+
+          changed_services=$(echo $changed_services | xargs)
+          echo "Changed services after removing alert-manager: $changed_services"
+          if [ -n "$changed_services" ]; then
+            $GITHUB_WORKSPACE/build/pai_build.py push \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s $changed_services
+          fi
 
       - name: Push Images of Changed Service to GHCR
         if: steps.check.outputs.has_changed == 'true'
@@ -158,21 +177,24 @@ jobs:
             $GITHUB_WORKSPACE/build/pai_build.py push \
               -c $GITHUB_WORKSPACE/config/cluster-configuration \
               -s alert-manager \
-              -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring \
+              -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring \
               --docker-registry ghcr.io \
               --docker-namespace ${GITHUB_REPOSITORY_OWNER} \
               --docker-username ${{ github.actor }} \
               --docker-password ${{ secrets.GITHUB_TOKEN }}
           fi
 
+          changed_services=$(echo $changed_services | xargs)
           echo "Changed services after removing alert-manager: $changed_services"
-          $GITHUB_WORKSPACE/build/pai_build.py push \
-            -c $GITHUB_WORKSPACE/config/cluster-configuration \
-            -s $changed_services \
-            --docker-registry ghcr.io \
-            --docker-namespace ${GITHUB_REPOSITORY_OWNER} \
-            --docker-username ${{ github.actor }} \
-            --docker-password ${{ secrets.GITHUB_TOKEN }}
+          if [ -n "$changed_services" ]; then
+            $GITHUB_WORKSPACE/build/pai_build.py push \
+              -c $GITHUB_WORKSPACE/config/cluster-configuration \
+              -s $changed_services \
+              --docker-registry ghcr.io \
+              --docker-namespace ${GITHUB_REPOSITORY_OWNER} \
+              --docker-username ${{ github.actor }} \
+              --docker-password ${{ secrets.GITHUB_TOKEN }}
+          fi
 
       - name: Azure CLI get credentials and deploy
         if: steps.check.outputs.has_changed == 'true'

build/core/build_utility.py

@@ -143,7 +143,7 @@ def docker_login(self):
 
     def docker_image_build(self, image_name, dockerfile_path, build_path):
         if self.build_nocache:
-            cmd = "docker build --no-cache -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path)
+            cmd = "docker build --no-cache --pull -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path)
         else:
             cmd = "docker build -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path)
         execute_shell(cmd)

src/alert-manager/src/cluster-utilization/send_alert.py

@@ -402,13 +402,24 @@ def main():
         trace_id = str(uuid.uuid4())
         lucia_url = os.environ.get("LUCIA_URL")
         # collect cluster gpu usage information
-        message = collect_report_from_lucia(
-            lucia_url,
-            os.environ.get("LUCIA_BEARER_TOKEN"),
-            trace_id
-        )
+        try:
+            message = collect_report_from_lucia(
+                lucia_url,
+                os.environ.get("LUCIA_BEARER_TOKEN"),
+                trace_id
+            )
+        except Exception:
+            logging.exception("Failed to collect report from Lucia")
+            return
         alerts = generate_alerts_from_lucia(message, lucia_url, trace_id)
-    send_alert(PAI_URI, alerts)
+    else:
+        logging.error(f"Unknown REPORT_SOURCE: {REPORT_SOURCE}")
+        return
+
+    try:
+        send_alert(PAI_URI, alerts)
+    except Exception as e:
+        logging.error(f"Failed to send alerts: {e}")
 
 if __name__ == "__main__":
     logging.basicConfig(