orchestratord test: Introduce upgrade scenario

ci/nightly/pipeline.template.yml

@@ -2307,35 +2307,83 @@ steps:
       - id: orchestratord-defaults
         label: "Orchestratord test (defaults for properties)"
         depends_on: build-aarch64
-        timeout_in_minutes: 180
+        timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
-              args: [--scenario=defaults, --recreate-cluster]
+              args: [--action=noop, --properties=defaults, --recreate-cluster]
               ci-builder: stable
         agents:
           queue: hetzner-aarch64-16cpu-32gb
 
-      - id: orchestratord-all
-        label: "Orchestratord test (every property)"
+      - id: orchestratord-individual
+        label: "Orchestratord test (individual properties)"
         depends_on: build-aarch64
-        timeout_in_minutes: 180
+        timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
-              args: [--scenario=all, --recreate-cluster]
+              args: [--action=noop, --properties=individual, --recreate-cluster]
               ci-builder: stable
         agents:
           queue: hetzner-aarch64-16cpu-32gb
 
       - id: orchestratord-combine
         label: "Orchestratord test (combine properties)"
         depends_on: build-aarch64
-        timeout_in_minutes: 180
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              args: [--action=noop, --properties=combine, --runtime=3600, --recreate-cluster]
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-upgrade-individual
+        label: "Orchestratord test (upgrade, individual props)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              args: [--action=upgrade, --properties=individual, --runtime=3600, --recreate-cluster]
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+
+      - id: orchestratord-upgrade-combine
+        label: "Orchestratord test (upgrade, combine props)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              args: [--action=upgrade, --properties=combine, --runtime=3600, --recreate-cluster]
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+
+      - id: orchestratord-upgrade-chain-individual
+        label: "Orchestratord test (upgrade chain, individual props)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              args: [--action=upgrade-chain, --properties=individual, --runtime=3600, --recreate-cluster]
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+
+      - id: orchestratord-upgrade-chain-combine
+        label: "Orchestratord test (upgrade chain, combine props)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
-              args: [--scenario=combine, --runtime=7200, --recreate-cluster]
+              args: [--action=upgrade-chain, --properties=combine, --runtime=3600, --recreate-cluster]
               ci-builder: stable
         agents:
           queue: hetzner-aarch64-16cpu-32gb

ci/plugins/mzcompose/hooks/command

@@ -134,15 +134,13 @@ cleanup() {
       ci_unimportant_heading "orchestratord test: Uploading logs..."
       K8S_CLUSTER_NAME=kind
       K8S_CONTEXT="kind-$K8S_CLUSTER_NAME"
-      for pod in $(kubectl get pods -o name | grep -v -E 'kubernetes|minio|cockroach|redpanda'); do
-          kubectl logs --prefix=true "$pod" &>> kubectl-get-logs.log || true
-          kubectl logs --previous --prefix=true "$pod" &>> kubectl-get-logs-previous.log || true
+      for pod in $(kubectl get pods -o name -n materialize | grep -v -E 'kubernetes|minio|cockroach|redpanda'); do
+          kubectl logs --prefix=true "$pod" -n materialize &>> kubectl-get-logs.log || true
+          kubectl logs --previous --prefix=true "$pod" -n materialize &>> kubectl-get-logs-previous.log || true
       done
-      kubectl get events > kubectl-get-events.log || true
-      kubectl get all > kubectl-get-all.log || true
-      kubectl get events > kubectl-get-events.log || true
-      kubectl get all > kubectl-get-all.log || true
-      kubectl describe all | awk '
+      kubectl get events -n materialize > kubectl-get-events.log || true
+      kubectl get all -n materialize > kubectl-get-all.log || true
+      kubectl describe all -n materialize | awk '
         BEGIN { redact=0 }
         /^[[:space:]]*Environment:/ {
           indent = match($0, /[^ ]/) - 1
@@ -160,13 +158,35 @@ cleanup() {
         }
         { print }
       ' > kubectl-describe-all.log || true
-      kubectl get pods -o wide > kubectl-pods-with-nodes.log || true
+      kubectl get pods -o wide -n materialize > kubectl-pods-with-nodes.log || true
 
-      kubectl -n kube-system get events > kubectl-get-events-kube-system.log || true
-      kubectl -n kube-system get all > kubectl-get-all-kube-system.log || true
-      kubectl -n kube-system describe all > kubectl-describe-all-kube-system.log || true
+      for pod in $(kubectl get pods -o name -n materialize-environment | grep -v -E 'kubernetes|minio|cockroach|redpanda'); do
+          kubectl logs --prefix=true "$pod" -n materialize-environment &>> kubectl-get-logs-environment.log || true
+          kubectl logs --previous --prefix=true "$pod" -n materialize-environment &>> kubectl-get-logs-previous-environment.log || true
+      done
+      kubectl get events -n materialize-environment > kubectl-get-events-environment.log || true
+      kubectl get all -n materialize-environment > kubectl-get-all-environment.log || true
+      kubectl describe all -n materialize-environment | awk '
+        BEGIN { redact=0 }
+        /^[[:space:]]*Environment:/ {
+          indent = match($0, /[^ ]/) - 1
+          print substr($0, 1, indent) "Environment: [REDACTED]"
+          redact = 1
+          next
+        }
+        redact {
+          current_indent = match($0, /[^ ]/) - 1
+          if (current_indent <= indent || NF == 0) {
+            redact = 0
+          } else {
+            next
+          }
+        }
+        { print }
+      ' > kubectl-describe-all-environment.log || true
+      kubectl get pods -o wide -n materialize-environment > kubectl-pods-with-nodes-environment.log || true
 
-      mapfile -t artifacts < <(printf "kubectl-get-logs.log\nkubectl-get-logs-previous.log\nkubectl-get-events.log\nkubectl-get-all.log\nkubectl-describe-all.log\nkubectl-pods-with-nodes.log\nkubectl-get-events-kube-system.log\nkubectl-get-all-kube-system.log\nkubectl-describe-all-kube-system.log\nkail-output.log\n")
+      mapfile -t artifacts < <(printf "kubectl-get-logs.log\nkubectl-get-logs-previous.log\nkubectl-get-events.log\nkubectl-get-all.log\nkubectl-describe-all.log\nkubectl-pods-with-nodes.log\nkubectl-get-logs-environment.log\nkubectl-get-logs-previous-environment.log\nkubectl-get-events-environment.log\nkubectl-get-all-environment.log\nkubectl-describe-all-environment.log\nkubectl-pods-with-nodes-environment.log\n")
       artifacts_str=$(IFS=";"; echo "${artifacts[*]}")
       buildkite-agent artifact upload "$artifacts_str"
       unset artifacts

misc/python/materialize/version_list.py

@@ -66,6 +66,26 @@ def get_self_managed_versions() -> list[MzVersion]:
     return sorted(result)
 
 
+BAD_SELF_MANAGED_VERSIONS = {
+    MzVersion.parse_mz("v0.130.0"),
+    MzVersion.parse_mz("v0.130.1"),
+    MzVersion.parse_mz("v0.130.2"),
+    MzVersion.parse_mz("v0.130.3"),
+    MzVersion.parse_mz("v0.130.4"),
+}
+
+
+def get_all_self_managed_versions() -> list[MzVersion]:
+    result = set()
+    for entry in yaml.safe_load(
+        requests.get("https://materializeinc.github.io/materialize/index.yaml").text
+    )["entries"]["materialize-operator"]:
+        version = MzVersion.parse_mz(entry["appVersion"])
+        if not version.prerelease and version not in BAD_SELF_MANAGED_VERSIONS:
+            result.add(version)
+    return sorted(result)
+
+
 # not released on Docker
 INVALID_VERSIONS = {
     MzVersion.parse_mz("v0.52.1"),