diff --git a/content/en/docs/ambient/install/multicluster/_index.md b/content/en/docs/ambient/install/multicluster/_index.md index 1bcb2c7bf58d..037ce91d4349 100644 --- a/content/en/docs/ambient/install/multicluster/_index.md +++ b/content/en/docs/ambient/install/multicluster/_index.md @@ -59,6 +59,14 @@ the current state and limitations of this feature. **If a service's waypoint is marked as global, that service will also be global** - This can lead to unintended cross-cluster traffic if not managed carefully + - The solution to this issue is tracked [here](https://github.com/istio/istio/issues/57710) + +#### Load Distribution on Remote Network + +**Traffic going to a remote network is not equally distributed between endpoints** + - When failing over to a remote network, a single endpoint on a remote network may get a disproportionate number of requests + due to multiplexing of HTTP requests and connection pooling + - The solution to this issue is tracked [here](https://github.com/istio/istio/issues/58039) #### Gateway Limitations diff --git a/content/en/docs/ambient/install/multicluster/common.sh b/content/en/docs/ambient/install/multicluster/common.sh index 37db8af06043..b0d9735f821c 100644 --- a/content/en/docs/ambient/install/multicluster/common.sh +++ b/content/en/docs/ambient/install/multicluster/common.sh @@ -19,6 +19,7 @@ _set_kube_vars source content/en/docs/ambient/install/multicluster/verify/snips.sh +source content/en/docs/ambient/install/multicluster/failover/snips.sh # set_single_network_vars initializes all variables for a single network config. function set_single_network_vars @@ -156,6 +157,52 @@ function verify_load_balancing _verify_contains snip_verifying_crosscluster_traffic_3 "$EXPECTED_RESPONSE_FROM_CLUSTER2" } +function deploy_waypoints +{ + # Deploy waypoints in both clusters and wait until they are up and running + snip_deploy_waypoint_proxy_1 + _wait_for_deployment sample waypoint "${CTX_CLUSTER1}" + _wait_for_deployment sample waypoint "${CTX_CLUSTER2}" + + # Label HelloWorld service to use the newly deployed waypoints + snip_deploy_waypoint_proxy_4 + # Mark waypoint service as global + snip_deploy_waypoint_proxy_5 +} + +function configure_locality_failover +{ + echo "Deploying locality failover configuration" + snip_configure_locality_failover_1 + snip_configure_locality_failover_2 +} + +function verify_traffic_local +{ + local EXPECTED_RESPONSE_FROM_CLUSTER1="Hello version: v1, instance:" + local EXPECTED_RESPONSE_FROM_CLUSTER2="Hello version: v2, instance:" + + echo "Verifying traffic stays in ${CTX_CLUSTER1}" + _verify_contains snip_verify_traffic_stays_in_local_cluster_1 "$EXPECTED_RESPONSE_FROM_CLUSTER1" + + echo "Verifying traffic stays in ${CTX_CLUSTER2}" + _verify_contains snip_verify_traffic_stays_in_local_cluster_3 "$EXPECTED_RESPONSE_FROM_CLUSTER2" +} + +function break_cluster1 +{ + echo "Breaking ${CTX_CLUSTER1}" + snip_verify_failover_to_another_cluster_1 +} + +function verify_failover +{ + local EXPECTED_RESPONSE_FROM_CLUSTER2="Hello version: v2, instance:" + + echo "Verifying that traffic from ${CTX_CLUSTER1} fails over to ${CTX_CLUSTER2}" + _verify_contains snip_verify_failover_to_another_cluster_2 "$EXPECTED_RESPONSE_FROM_CLUSTER2" +} + # For Helm multi-cluster installation steps function create_istio_system_ns diff --git a/content/en/docs/ambient/install/multicluster/failover/index.md b/content/en/docs/ambient/install/multicluster/failover/index.md new file mode 100644 index 000000000000..6cdac12e4738 --- /dev/null +++ b/content/en/docs/ambient/install/multicluster/failover/index.md @@ -0,0 +1,192 @@ +--- +title: Configure failover behavior in multicluster ambient installation +description: Configure outlier detection and failover behavior in ambient multicluster ambient mesh using waypoints. +weight: 70 +keywords: [kubernetes,multicluster,ambient] +test: yes +owner: istio/wg-environments-maintainers +prev: /docs/ambient/install/multicluster/verify +--- +Follow this guide to customize failover behavior in your ambient multicluster Istio installation using waypoint proxies. + +Before proceeding, be sure to complete ambient multicluster Istio installation following one of the +[multicluster installation guides](/docs/ambient/install/multicluster) and verify that the installation is working properly. + +In this guide, we will build on top of the `HelloWorld` application used to verify the multicluster installation. We will +configure locality failover for the `HelloWorld` service to prefer endpoints in the cluster local to the client using a +`DestinationRule` and will deploy a waypoint proxy to enforce the configuration. + +## Deploy waypoint proxy + +In order to configure outlier detection and customize failover behavior for the service we need a waypoint proxy. To begin, +deploy waypoint proxy to each cluster in the mesh: + +{{< text bash >}} +$ istioctl --context "${CTX_CLUSTER1}" waypoint apply --name waypoint --for service -n sample --wait +$ istioctl --context "${CTX_CLUSTER2}" waypoint apply --name waypoint --for service -n sample --wait +{{< /text >}} + +Confirm the status of the waypoint proxy deployment on `cluster1`: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER1}" get deployment waypoint --namespace sample +NAME READY UP-TO-DATE AVAILABLE AGE +waypoint 1/1 1 1 137m +{{< /text >}} + +Confirm the status of the waypoint proxy deployment on `cluster2`: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER2}" get deployment waypoint --namespace sample +NAME READY UP-TO-DATE AVAILABLE AGE +waypoint 1/1 1 1 138m +{{< /text >}} + +Wait until all waypoint proxies are ready. + +Configure `HelloWorld` service in each cluster to use the waypoint proxy: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER1}" label svc helloworld -n sample istio.io/use-waypoint=waypoint +$ kubectl --context "${CTX_CLUSTER2}" label svc helloworld -n sample istio.io/use-waypoint=waypoint +{{< /text >}} + +Finally, and this step is specific to multicluster deployment of waypoint proxies, mark the waypoint proxy service in each +cluster as global, just like you did earlier with the `HelloWorld` service: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER1}" label svc waypoint -n sample istio.io/global=true +$ kubectl --context "${CTX_CLUSTER2}" label svc waypoint -n sample istio.io/global=true +{{< /text >}} + +The `HelloWorld` service in both clusters is now configured to use waypoint proxies, but waypoint proxies don't do anything +useful yet. + +## Configure locality failover + +To configure locality failover create and apply a `DestinationRule` in `cluster1`: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER1}" apply -n sample -f - <}} + +Apply the same `DestinationRule` in `cluster2` as well: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER2}" apply -n sample -f - <}} + +This `DestinationRule` configures the following: + +- [Outlier detection](/docs/reference/config/networking/destination-rule/#OutlierDetection) for the `HelloWorld` service. + This instructs waypoint proxies how to identify when endpoints for a service are unhealthy. It's required for failover + to function properly. + +- [Failover priority](/docs/reference/config/networking/destination-rule/#LocalityLoadBalancerSetting) that instructs + waypoint proxy how to prioritize endpoints when routing requests. In this example, waypoint proxy will prefer endpoints + in the same cluster over endpoints in other clusters. + +With these policies in place, waypoint proxies will prefer endpoints in the same cluster as the waypoint proxy when they +are available and considered healthy based on the outlier detection configuration. + +## Verify traffic stays in local cluster + +Send request from the `curl` pods on `cluster1` to the `HelloWorld` service: + +{{< text bash >}} +$ kubectl exec --context "${CTX_CLUSTER1}" -n sample -c curl \ + "$(kubectl get pod --context "${CTX_CLUSTER1}" -n sample -l \ + app=curl -o jsonpath='{.items[0].metadata.name}')" \ + -- curl -sS helloworld.sample:5000/hello +{{< /text >}} + +Now, if you repeat this request several times and verify that the `HelloWorld` version should always be `v1` because the +traffic stays in `cluster1`: + +{{< text plain >}} +Hello version: v1, instance: helloworld-v1-954745fd-z6qcn +Hello version: v1, instance: helloworld-v1-954745fd-z6qcn +... +{{< /text >}} + +Similarly, send request from `curl` pods on `cluster2` several times: + +{{< text bash >}} +$ kubectl exec --context "${CTX_CLUSTER2}" -n sample -c curl \ + "$(kubectl get pod --context "${CTX_CLUSTER2}" -n sample -l \ + app=curl -o jsonpath='{.items[0].metadata.name}')" \ + -- curl -sS helloworld.sample:5000/hello +{{< /text >}} + +You should see that all requests are processed in `cluster2` by looking at the version in the response: + +{{< text plain >}} +Hello version: v2, instance: helloworld-v2-7b768b9bbd-7zftm +Hello version: v2, instance: helloworld-v2-7b768b9bbd-7zftm +... +{{< /text >}} + +## Verify failover to another cluster + +To verify that failover to remote cluster works simulate `HelloWorld` service outage in `cluster1` by scaling down +deployment: + +{{< text bash >}} +$ kubectl --context "${CTX_CLUSTER1}" scale --replicas=0 deployment/helloworld-v1 -n sample +{{< /text >}} + +Send request from the `curl` pods on `cluster1` to the `HelloWorld` service again: + +{{< text bash >}} +$ kubectl exec --context "${CTX_CLUSTER1}" -n sample -c curl \ + "$(kubectl get pod --context "${CTX_CLUSTER1}" -n sample -l \ + app=curl -o jsonpath='{.items[0].metadata.name}')" \ + -- curl -sS helloworld.sample:5000/hello +{{< /text >}} + +This time you should see that the request is processed by `HelloWorld` service in `cluster2` because there are no +available endpoints in `cluster1`: + +{{< text plain >}} +Hello version: v2, instance: helloworld-v2-7b768b9bbd-7zftm +Hello version: v2, instance: helloworld-v2-7b768b9bbd-7zftm +... +{{< /text >}} + +**Congratulations!** You successfully configuration locality failover in Istio ambient multicluster deployment! diff --git a/content/en/docs/ambient/install/multicluster/failover/snips.sh b/content/en/docs/ambient/install/multicluster/failover/snips.sh new file mode 100644 index 000000000000..a2f16263a2d8 --- /dev/null +++ b/content/en/docs/ambient/install/multicluster/failover/snips.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2153,SC2155,SC2164 + +# Copyright Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#################################################################################################### +# WARNING: THIS IS AN AUTO-GENERATED FILE, DO NOT EDIT. PLEASE MODIFY THE ORIGINAL MARKDOWN FILE: +# docs/ambient/install/multicluster/failover/index.md +#################################################################################################### + +snip_deploy_waypoint_proxy_1() { +istioctl --context "${CTX_CLUSTER1}" waypoint apply --name waypoint --for service -n sample --wait +istioctl --context "${CTX_CLUSTER2}" waypoint apply --name waypoint --for service -n sample --wait +} + +snip_deploy_waypoint_proxy_2() { +kubectl --context "${CTX_CLUSTER1}" get deployment waypoint --namespace sample +} + +! IFS=$'\n' read -r -d '' snip_deploy_waypoint_proxy_2_out <<\ENDSNIP +NAME READY UP-TO-DATE AVAILABLE AGE +waypoint 1/1 1 1 137m +ENDSNIP + +snip_deploy_waypoint_proxy_3() { +kubectl --context "${CTX_CLUSTER2}" get deployment waypoint --namespace sample +} + +! IFS=$'\n' read -r -d '' snip_deploy_waypoint_proxy_3_out <<\ENDSNIP +NAME READY UP-TO-DATE AVAILABLE AGE +waypoint 1/1 1 1 138m +ENDSNIP + +snip_deploy_waypoint_proxy_4() { +kubectl --context "${CTX_CLUSTER1}" label svc helloworld -n sample istio.io/use-waypoint=waypoint +kubectl --context "${CTX_CLUSTER2}" label svc helloworld -n sample istio.io/use-waypoint=waypoint +} + +snip_deploy_waypoint_proxy_5() { +kubectl --context "${CTX_CLUSTER1}" label svc waypoint -n sample istio.io/global=true +kubectl --context "${CTX_CLUSTER2}" label svc waypoint -n sample istio.io/global=true +} + +snip_configure_locality_failover_1() { +kubectl --context "${CTX_CLUSTER1}" apply -n sample -f - < +## Next steps + +Configure [locality failover](/docs/ambient/install/multicluster/failover) for your multicluster deployment.