diff --git a/.github/workflows/build-all.yaml b/.github/workflows/build-all.yaml index 43ae8330..2ecbb55f 100644 --- a/.github/workflows/build-all.yaml +++ b/.github/workflows/build-all.yaml @@ -100,7 +100,8 @@ jobs: -n \ -c $GITHUB_WORKSPACE/config/cluster-configuration \ -s alert-manager \ - -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring + -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring + continue fi echo "Building service: $service" if python3 $GITHUB_WORKSPACE/build/pai_build.py build \ diff --git a/.github/workflows/build-deploy-changes.yaml b/.github/workflows/build-deploy-changes.yaml index e61a0b5a..4e1360e5 100644 --- a/.github/workflows/build-deploy-changes.yaml +++ b/.github/workflows/build-deploy-changes.yaml @@ -123,24 +123,43 @@ jobs: $GITHUB_WORKSPACE/build/pai_build.py build \ -n \ -c $GITHUB_WORKSPACE/config/cluster-configuration \ - -s alert-manager - -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring + -s alert-manager \ + -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring fi + changed_services=$(echo $changed_services | xargs) echo "Changed services after removing alert-manager: $changed_services" - $GITHUB_WORKSPACE/build/pai_build.py build \ - -n \ - -c $GITHUB_WORKSPACE/config/cluster-configuration \ - -s $changed_services + if [ -n "$changed_services" ]; then + $GITHUB_WORKSPACE/build/pai_build.py build \ + -n \ + -c $GITHUB_WORKSPACE/config/cluster-configuration \ + -s $changed_services + fi - name: Push Images of Changed Services to ACR if: steps.check.outputs.has_changed == 'true' + shell: bash run: | changed_services="${{ steps.changes.outputs.folders }}" echo "Pushing: $changed_services" - $GITHUB_WORKSPACE/build/pai_build.py push \ - -c $GITHUB_WORKSPACE/config/cluster-configuration \ - -s $changed_services + if [[ "$changed_services" == *"alert-manager"* ]]; then + echo "alert-manager is in the changed services" + changed_services=$(echo $changed_services | sed 's/alert-manager//g') + # push specific images in alert-manager to ACR + echo "Pushing specific alert-manager images to ACR" + $GITHUB_WORKSPACE/build/pai_build.py push \ + -c $GITHUB_WORKSPACE/config/cluster-configuration \ + -s alert-manager \ + -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring + fi + + changed_services=$(echo $changed_services | xargs) + echo "Changed services after removing alert-manager: $changed_services" + if [ -n "$changed_services" ]; then + $GITHUB_WORKSPACE/build/pai_build.py push \ + -c $GITHUB_WORKSPACE/config/cluster-configuration \ + -s $changed_services + fi - name: Push Images of Changed Service to GHCR if: steps.check.outputs.has_changed == 'true' @@ -158,21 +177,24 @@ jobs: $GITHUB_WORKSPACE/build/pai_build.py push \ -c $GITHUB_WORKSPACE/config/cluster-configuration \ -s alert-manager \ - -i abnormal-detector,alert-handler,alert-parser,cert-expiration-checker,cluster-utilization,job-data-recorder,job-status-change-notification,node-failure-detection,node-issue-classifier,nvidia-gpu-low-perf-fixer,redis-monitoring \ + -i abnormal-detector alert-handler alert-parser cert-expiration-checker cluster-utilization job-data-recorder job-status-change-notification node-failure-detection node-issue-classifier nvidia-gpu-low-perf-fixer redis-monitoring \ --docker-registry ghcr.io \ --docker-namespace ${GITHUB_REPOSITORY_OWNER} \ --docker-username ${{ github.actor }} \ --docker-password ${{ secrets.GITHUB_TOKEN }} fi + changed_services=$(echo $changed_services | xargs) echo "Changed services after removing alert-manager: $changed_services" - $GITHUB_WORKSPACE/build/pai_build.py push \ - -c $GITHUB_WORKSPACE/config/cluster-configuration \ - -s $changed_services \ - --docker-registry ghcr.io \ - --docker-namespace ${GITHUB_REPOSITORY_OWNER} \ - --docker-username ${{ github.actor }} \ - --docker-password ${{ secrets.GITHUB_TOKEN }} + if [ -n "$changed_services" ]; then + $GITHUB_WORKSPACE/build/pai_build.py push \ + -c $GITHUB_WORKSPACE/config/cluster-configuration \ + -s $changed_services \ + --docker-registry ghcr.io \ + --docker-namespace ${GITHUB_REPOSITORY_OWNER} \ + --docker-username ${{ github.actor }} \ + --docker-password ${{ secrets.GITHUB_TOKEN }} + fi - name: Azure CLI get credentials and deploy if: steps.check.outputs.has_changed == 'true' diff --git a/build/core/build_utility.py b/build/core/build_utility.py index 68ce7746..72b43ab4 100644 --- a/build/core/build_utility.py +++ b/build/core/build_utility.py @@ -143,7 +143,7 @@ def docker_login(self): def docker_image_build(self, image_name, dockerfile_path, build_path): if self.build_nocache: - cmd = "docker build --no-cache -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path) + cmd = "docker build --no-cache --pull -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path) else: cmd = "docker build -t {0} -f {1} {2}".format(image_name, dockerfile_path, build_path) execute_shell(cmd) diff --git a/src/alert-manager/src/cluster-utilization/send_alert.py b/src/alert-manager/src/cluster-utilization/send_alert.py index 42aaad9f..65ce7cc5 100644 --- a/src/alert-manager/src/cluster-utilization/send_alert.py +++ b/src/alert-manager/src/cluster-utilization/send_alert.py @@ -402,13 +402,24 @@ def main(): trace_id = str(uuid.uuid4()) lucia_url = os.environ.get("LUCIA_URL") # collect cluster gpu usage information - message = collect_report_from_lucia( - lucia_url, - os.environ.get("LUCIA_BEARER_TOKEN"), - trace_id - ) + try: + message = collect_report_from_lucia( + lucia_url, + os.environ.get("LUCIA_BEARER_TOKEN"), + trace_id + ) + except Exception: + logging.exception("Failed to collect report from Lucia") + return alerts = generate_alerts_from_lucia(message, lucia_url, trace_id) - send_alert(PAI_URI, alerts) + else: + logging.error(f"Unknown REPORT_SOURCE: {REPORT_SOURCE}") + return + + try: + send_alert(PAI_URI, alerts) + except Exception as e: + logging.error(f"Failed to send alerts: {e}") if __name__ == "__main__": logging.basicConfig(