feat: add pathways job and update workloads version to latest

platforms/gke/base/_shared_config/workloads_variables.tf

@@ -17,7 +17,7 @@ locals {
 }
 
 variable "custom_metrics_adapter_version" {
-  default     = "0.16.2"
+  default     = "0.16.7"
   description = "Version of Custom Metrics Adapter (https://github.com/GoogleCloudPlatform/k8s-stackdriver) to install."
   type        = string
 }
@@ -29,31 +29,37 @@ variable "inference_gateway_kubernetes_namespace" {
 }
 
 variable "inference_gateway_version" {
-  default     = "1.1.0"
+  default     = "1.5.0"
   description = "Version of Gateway API Inference Extension (https://github.com/kubernetes-sigs/gateway-api-inference-extension) to install."
   type        = string
 }
 
 variable "jobset_version" {
-  default     = "0.10.1"
+  default     = "0.12.0"
   description = "Version of JobSet (https://github.com/kubernetes-sigs/jobset/) to install."
   type        = string
 }
 
 variable "kuberay_version" {
-  default     = "1.5.1"
+  default     = "1.6.1"
   description = "Version of KubeRay (https://github.com/ray-project/kuberay) to install."
   type        = string
 }
 
 variable "kueue_version" {
-  default     = "0.14.4"
-  description = "Version of Kueue (https://kueue.sigs.k8s.io/) to install."
+  default     = "0.17.2"
+  description = "Version of Kueue (https://github.com/kubernetes-sigs/kueue) to install."
   type        = string
 }
 
 variable "lws_version" {
-  default     = "0.7.0"
+  default     = "0.8.0"
   description = "Version of LeaderWorkerSet (LWS) (https://github.com/kubernetes-sigs/lws/) to install."
   type        = string
 }
+
+variable "pathways_version" {
+  default     = "0.1.4"
+  description = "Version of Pathways (https://github.com/google/pathways-job) to install."
+  type        = string
+}

platforms/gke/base/core/initialize/local_file.tf

@@ -279,6 +279,7 @@ resource "local_file" "shared_config_workloads_auto_tfvars" {
       kuberay_version                        = var.kuberay_version
       kueue_version                          = var.kueue_version
       lws_version                            = var.lws_version
+      pathways_version                       = var.pathways_version
     }
   )
   file_permission = "0644"

platforms/gke/base/core/workloads/pathways/_cluster.auto.tfvars

@@ -0,0 +1 @@
+../../../_shared_config/cluster.auto.tfvars

platforms/gke/base/core/workloads/pathways/_cluster_variables.tf

@@ -0,0 +1 @@
+../../../_shared_config/cluster_variables.tf

platforms/gke/base/core/workloads/pathways/_platform.auto.tfvars

@@ -0,0 +1 @@
+../../../_shared_config/platform.auto.tfvars

platforms/gke/base/core/workloads/pathways/_platform_variables.tf

@@ -0,0 +1 @@
+../../../_shared_config/platform_variables.tf

platforms/gke/base/core/workloads/pathways/_workloads.auto.tfvars

@@ -0,0 +1 @@
+../../../_shared_config/workloads.auto.tfvars

platforms/gke/base/core/workloads/pathways/_workloads_variables.tf

@@ -0,0 +1 @@
+../../../_shared_config/workloads_variables.tf

platforms/gke/base/core/workloads/pathways/main.tf

@@ -0,0 +1,129 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+locals {
+  kubeconfig_directory = "${path.module}/../../../kubernetes/kubeconfig"
+  kubeconfig_file      = "${local.kubeconfig_directory}/${local.kubeconfig_file_name}"
+
+  manifests_directory         = "${local.namespace_directory}/pathways-system"
+  namespace_directory         = "${local.manifests_directory_root}/namespace"
+  version_manifests_directory = "${path.module}/manifests/pathways-${var.jobset_version}"
+}
+
+data "local_file" "kubeconfig" {
+  filename = local.kubeconfig_file
+}
+
+resource "terraform_data" "namespace" {
+  input = {
+    manifests_dir = local.namespace_directory
+  }
+
+  provisioner "local-exec" {
+    command     = <<EOT
+mkdir -p "${self.input.manifests_dir}" && \
+cp -r templates/namespace-pathways-system.yaml "${self.input.manifests_dir}/"
+EOT
+    interpreter = ["bash", "-c"]
+    working_dir = path.module
+  }
+
+  triggers_replace = {
+    manifests_dir = local.namespace_directory
+  }
+}
+
+module "kubectl_apply_namespace" {
+  depends_on = [
+    terraform_data.namespace,
+  ]
+
+  source = "../../../modules/kubectl_apply"
+
+  apply_server_side           = true
+  delete_timeout              = "300s"
+  error_on_delete_failure     = false
+  kubeconfig_file             = data.local_file.kubeconfig.filename
+  manifest                    = "${local.namespace_directory}/namespace-pathways-system.yaml"
+  manifest_includes_namespace = true
+}
+
+resource "terraform_data" "manifests" {
+  input = {
+    manifests_dir         = local.manifests_directory
+    version               = var.pathways_version
+    version_manifests_dir = local.version_manifests_directory
+  }
+
+  provisioner "local-exec" {
+    command     = <<EOT
+mkdir -p "${self.input.version_manifests_dir}" && \
+mkdir -p "${self.input.manifests_dir}" && \
+wget https://github.com/google/pathways-job/releases/download/v${self.input.version}/install.yaml -O "${self.input.version_manifests_dir}/manifests.yaml" && \
+cp -r templates/workload/* "${self.input.version_manifests_dir}/" && \
+cp -r "${self.input.version_manifests_dir}"/* "${self.input.manifests_dir}/"
+EOT
+    interpreter = ["bash", "-c"]
+    working_dir = path.module
+  }
+
+  triggers_replace = {
+    manifests_dir         = local.manifests_directory
+    version               = var.pathways_version
+    version_manifests_dir = local.version_manifests_directory
+  }
+}
+
+resource "terraform_data" "manifests_ap" {
+  depends_on = [
+    terraform_data.manifests,
+  ]
+
+  for_each = toset(var.cluster_autopilot_enabled ? ["remove-resource-type"] : [])
+
+  input = {
+    manifests_dir         = local.manifests_directory
+    version               = var.pathways_version
+    version_manifests_dir = local.version_manifests_directory
+  }
+  provisioner "local-exec" {
+    command     = <<EOT
+sed --expression='/- path: patch\/gke-managed-components-toleration.yaml$/,+4d' --in-place ${self.input.version_manifests_dir}/kustomization.yaml && \
+sed --expression='/- path: patch\/gke-managed-components-toleration.yaml$/,+4d' --in-place ${self.input.manifests_dir}/kustomization.yaml
+EOT
+    interpreter = ["bash", "-c"]
+    working_dir = path.module
+  }
+
+  triggers_replace = {
+    manifests_dir         = local.manifests_directory
+    version               = var.pathways_version
+    version_manifests_dir = local.version_manifests_directory
+  }
+}
+
+module "kubectl_apply_manifests" {
+  depends_on = [
+    module.kubectl_apply_namespace,
+    terraform_data.manifests_ap,
+  ]
+
+  source = "../../../modules/kubectl_apply"
+
+  apply_server_side           = true
+  kubeconfig_file             = data.local_file.kubeconfig.filename
+  manifest                    = local.version_manifests_directory
+  manifest_includes_namespace = true
+  use_kustomize               = true
+}

platforms/gke/base/core/workloads/pathways/templates/namespace-pathways-system.yaml

@@ -0,0 +1,22 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: pathways-job
+    control-plane: controller-manager
+  name: pathways-job-system

platforms/gke/base/core/workloads/pathways/templates/workload/kustomization.yaml

@@ -0,0 +1,24 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+patches:
+- path: patch/gke-managed-components-toleration.yaml
+  target:
+    group: apps
+    kind: Deployment
+    version: v1
+- path: patch/no-create-namespace.yaml
+
+resources:
+- manifests.yaml

platforms/gke/base/core/workloads/pathways/templates/workload/patch/gke-managed-components-toleration.yaml

@@ -0,0 +1,28 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  name: not-applicable
+spec:
+  template:
+    spec:
+      nodeSelector:
+        resource-type: system
+      tolerations:
+      - effect: NoSchedule
+        key: "components.gke.io/gke-managed-components"
+        operator: "Equal"
+        value: "true"

platforms/gke/base/core/workloads/pathways/templates/workload/patch/no-create-namespace.yaml

@@ -0,0 +1,19 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$patch: delete
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pathways-job-system

platforms/gke/base/core/workloads/pathways/versions.tf

@@ -0,0 +1,32 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_version = ">= 1.5.7"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "7.6.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "2.5.3"
+    }
+  }
+
+  provider_meta "google" {
+    module_name = "cloud-solutions/acp_gke_base_core_workloads_pathways_deploy-v1"
+  }
+}

platforms/gke/base/use-cases/inference-ref-arch/terraform/deploy-standard.sh

@@ -51,6 +51,7 @@ declare -a CORE_TERRASERVICES_APPLY=(
   "workloads/lws"
   "workloads/priority_class"
   "workloads/kueue"
+  "workloads/pathways"
 )
 CORE_TERRASERVICES_APPLY="${CORE_TERRASERVICES_APPLY[*]}" "${ACP_PLATFORM_CORE_DIR}/deploy.sh"
 

platforms/gke/base/use-cases/inference-ref-arch/terraform/teardown-standard.sh

@@ -68,6 +68,7 @@ done
 
 if [ "${ACP_TEARDOWN_CORE_PLATFORM}" = "true" ]; then
   declare -a CORE_TERRASERVICES_DESTROY=(
+    "workloads/pathways"
     "workloads/kueue"
     "workloads/priority_class"
     "workloads/lws"