diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index 9381d4eb7aca..332b0965524d 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -45,6 +45,20 @@ message AdditionalAddress { // or an empty list of :ref:`socket_options `, // it means no socket option will apply. core.v3.SocketOptionsOverride socket_options = 2; + + // TcpKeepaliveOverride wraps the TCP keepalive settings for the additional address. + // It provides a mechanism to override or disable TCP keepalive on a per-address basis. + message TcpKeepaliveOverride { + // Configures TCP keepalive for the additional address. + core.v3.TcpKeepalive tcp_keepalive = 1; + } + + // Configures TCP keepalive settings for the additional address. + // If specified, this will override the listener :ref:`tcp_keepalive ` configuration. + // If the :ref:`tcp_keepalive ` + // within the override is not specified, it means TCP keepalive settings will not be enabled + // for the additional address. + TcpKeepaliveOverride tcp_keepalive_override = 3; } // Listener list collections. Entries are ``Listener`` resources or references. @@ -53,7 +67,7 @@ message ListenerCollection { repeated xds.core.v3.CollectionEntry entries = 1; } -// [#next-free-field: 36] +// [#next-free-field: 37] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -390,6 +404,11 @@ message Listener { // Whether the listener bypasses configured overload manager actions. bool bypass_overload_manager = 35; + + // If set, then TCP keepalive settings are configured for the listener address + // and additional addresses. See :ref:`tcp_keepalive_override ` + // to override TCP keepalive settings for additional addresses. + core.v3.TcpKeepalive tcp_keepalive = 36; } // A placeholder proto so that users can explicitly configure the standard diff --git a/changelogs/current.yaml b/changelogs/current.yaml index f9afc69079d6..54c8f8f2e736 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -176,5 +176,11 @@ new_features: change: | Made the :ref:`credential injector filter ` work as an upstream filter. +- area: listener + change: | + Added support for configuring TCP keepalive settings on both primary and additional listener addresses + by setting :ref:`tcp_keepalive ` on the primary + address and :ref:`tcp_keepalive ` + on additional addresses. deprecated: diff --git a/source/common/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc index 9db574220cce..07eb3d82fa98 100644 --- a/source/common/listener_manager/listener_impl.cc +++ b/source/common/listener_manager/listener_impl.cc @@ -340,13 +340,12 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, quic_stat_names_(parent_.quicStatNames()), missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { - std::vector>> - address_opts_list; + std::vector address_opts_list; if (config.has_internal_listener()) { addresses_.emplace_back( std::make_shared(config.name())); - address_opts_list.emplace_back(std::ref(config.socket_options())); + address_opts_list.emplace_back( + Network::SocketOptionFactory::buildLiteralOptions(config.socket_options())); } else { // All the addresses should be same socket type, so get the first address's socket type is // enough. @@ -355,7 +354,16 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, auto address = std::move(address_or_error.value()); SET_AND_RETURN_IF_NOT_OK(checkIpv4CompatAddress(address, config.address()), creation_status); addresses_.emplace_back(address); - address_opts_list.emplace_back(std::ref(config.socket_options())); + auto opts = std::make_shared(); + auto keepalive_opts = std::make_shared(); + if (config.has_tcp_keepalive()) { + keepalive_opts = Network::SocketOptionFactory::buildTcpKeepaliveOptions( + Network::parseTcpKeepaliveConfig(config.tcp_keepalive())); + } + addListenSocketOptions(opts, keepalive_opts); + addListenSocketOptions( + opts, Network::SocketOptionFactory::buildLiteralOptions(config.socket_options())); + address_opts_list.emplace_back(opts); for (auto i = 0; i < config.additional_addresses_size(); i++) { if (socket_type_ != @@ -374,12 +382,29 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, checkIpv4CompatAddress(address, config.additional_addresses(i).address()), creation_status); addresses_.emplace_back(additional_address); + auto opts = std::make_shared(); + + if (config.additional_addresses(i).has_tcp_keepalive_override()) { + if (config.additional_addresses(i).tcp_keepalive_override().has_tcp_keepalive()) { + addListenSocketOptions( + opts, + Network::SocketOptionFactory::buildTcpKeepaliveOptions( + Network::parseTcpKeepaliveConfig( + config.additional_addresses(i).tcp_keepalive_override().tcp_keepalive()))); + } + } else if (config.has_tcp_keepalive()) { + addListenSocketOptions(opts, keepalive_opts); + } + if (config.additional_addresses(i).has_socket_options()) { - address_opts_list.emplace_back( - std::ref(config.additional_addresses(i).socket_options().socket_options())); + addListenSocketOptions( + opts, Network::SocketOptionFactory::buildLiteralOptions( + config.additional_addresses(i).socket_options().socket_options())); } else { - address_opts_list.emplace_back(std::ref(config.socket_options())); + addListenSocketOptions( + opts, Network::SocketOptionFactory::buildLiteralOptions(config.socket_options())); } + address_opts_list.emplace_back(opts); } } @@ -663,9 +688,7 @@ ListenerImpl::buildUdpListenerFactory(const envoy::config::listener::v3::Listene void ListenerImpl::buildListenSocketOptions( const envoy::config::listener::v3::Listener& config, - std::vector>>& - address_opts_list) { + std::vector& address_opts_list) { listen_socket_options_list_.insert(listen_socket_options_list_.begin(), addresses_.size(), nullptr); for (std::vectorempty()) { + addListenSocketOptions(listen_socket_options_list_[i], address_opts_list[i]); } if (socket_type_ == Network::Socket::Type::Datagram) { // Needed for recvmsg to return destination address in IP header. diff --git a/source/common/listener_manager/listener_impl.h b/source/common/listener_manager/listener_impl.h index 2f37b82b1dbb..cab1413925a1 100644 --- a/source/common/listener_manager/listener_impl.h +++ b/source/common/listener_manager/listener_impl.h @@ -404,8 +404,7 @@ class ListenerImpl final : public Network::ListenerConfig, absl::Status buildUdpListenerFactory(const envoy::config::listener::v3::Listener& config, uint32_t concurrency); void buildListenSocketOptions(const envoy::config::listener::v3::Listener& config, - std::vector>>& address_opts_list); + std::vector& address_opts_list); absl::Status createListenerFilterFactories(const envoy::config::listener::v3::Listener& config); absl::Status validateFilterChains(const envoy::config::listener::v3::Listener& config); absl::Status buildFilterChains(const envoy::config::listener::v3::Listener& config); diff --git a/source/common/network/socket_option_factory.h b/source/common/network/socket_option_factory.h index 555cfc04b235..a954c836feb8 100644 --- a/source/common/network/socket_option_factory.h +++ b/source/common/network/socket_option_factory.h @@ -6,6 +6,7 @@ #include "source/common/common/logger.h" #include "source/common/protobuf/protobuf.h" +#include "source/common/protobuf/utility.h" #include "absl/types/optional.h" @@ -19,6 +20,14 @@ struct TcpKeepaliveConfig { absl::optional keepalive_interval_; // Interval between probes, in ms }; +static inline Network::TcpKeepaliveConfig +parseTcpKeepaliveConfig(const envoy::config::core::v3::TcpKeepalive& options) { + return Network::TcpKeepaliveConfig{ + PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_probes, absl::optional()), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_time, absl::optional()), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_interval, absl::optional())}; +} + class SocketOptionFactory : Logger::Loggable { public: static std::unique_ptr diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 822712799971..30e03c16da6b 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -71,16 +71,6 @@ std::string addressToString(Network::Address::InstanceConstSharedPtr address) { return address->asString(); } -Network::TcpKeepaliveConfig -parseTcpKeepaliveConfig(const envoy::config::cluster::v3::Cluster& config) { - const envoy::config::core::v3::TcpKeepalive& options = - config.upstream_connection_options().tcp_keepalive(); - return Network::TcpKeepaliveConfig{ - PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_probes, absl::optional()), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_time, absl::optional()), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, keepalive_interval, absl::optional())}; -} - absl::StatusOr createProtocolOptionsConfig(const std::string& name, const ProtobufWkt::Any& typed_config, Server::Configuration::ProtocolOptionsFactoryContext& factory_context) { @@ -196,9 +186,10 @@ buildBaseSocketOptions(const envoy::config::cluster::v3::Cluster& cluster_config Network::SocketOptionFactory::buildIpFreebindOptions()); } if (cluster_config.upstream_connection_options().has_tcp_keepalive()) { - Network::Socket::appendOptions(base_options, - Network::SocketOptionFactory::buildTcpKeepaliveOptions( - parseTcpKeepaliveConfig(cluster_config))); + Network::Socket::appendOptions( + base_options, + Network::SocketOptionFactory::buildTcpKeepaliveOptions(Network::parseTcpKeepaliveConfig( + cluster_config.upstream_connection_options().tcp_keepalive()))); } return base_options; diff --git a/test/common/listener_manager/BUILD b/test/common/listener_manager/BUILD index 23917a437994..6d57f54fafb5 100644 --- a/test/common/listener_manager/BUILD +++ b/test/common/listener_manager/BUILD @@ -76,6 +76,7 @@ envoy_cc_test( shard_count = 5, deps = [ ":listener_manager_impl_test_lib", + "//envoy/network:socket_interface", "//source/common/api:os_sys_calls_lib", "//source/common/config:metadata_lib", "//source/common/listener_manager:active_raw_udp_listener_config", diff --git a/test/common/listener_manager/listener_manager_impl_test.cc b/test/common/listener_manager/listener_manager_impl_test.cc index 2ad053736efe..0eb750ac8401 100644 --- a/test/common/listener_manager/listener_manager_impl_test.cc +++ b/test/common/listener_manager/listener_manager_impl_test.cc @@ -11,6 +11,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/listener/v3/listener.pb.h" +#include "envoy/network/socket.h" #include "envoy/server/filter_config.h" #include "envoy/server/listener_manager.h" #include "envoy/stream_info/filter_state.h" @@ -6244,6 +6245,190 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, LiteralSockoptListenerEnabled) { EXPECT_EQ(1U, manager_->listeners().size()); } +TEST_P(ListenerManagerImplWithRealFiltersTest, ListenerKeepaliveEnabled) { + if (!ENVOY_SOCKET_SO_KEEPALIVE.hasValue()) { + return; // Keepalive is not supported on this platform. + } + + const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( + name: SockoptsListener + address: + socket_address: { address: 127.0.0.1, port_value: 1111 } + additional_addresses: + - address: + socket_address: { address: 127.0.0.1, port_value: 2222 } + enable_reuse_port: false + filter_chains: + - filters: [] + name: foo + tcp_keepalive: {} + )EOF"); + + // Second address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 1, + ListenerComponentFactory::BindType::NoReusePort); + + // First address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 1, + ListenerComponentFactory::BindType::NoReusePort); + + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_SO_KEEPALIVE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_SO_KEEPALIVE.option(), + /* expected_value */ 1, + /* expected_num_calls */ 2); + addOrUpdateListener(listener); + EXPECT_EQ(1U, manager_->listeners().size()); +} + +TEST_P(ListenerManagerImplWithRealFiltersTest, ListenerKeepaliveEnabledWithOpts) { + if (!ENVOY_SOCKET_SO_KEEPALIVE.hasValue()) { + return; // Keepalive is not supported on this platform. + } + + const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( + name: SockoptsListener + address: + socket_address: { address: 127.0.0.1, port_value: 1111 } + additional_addresses: + - address: + socket_address: { address: 127.0.0.1, port_value: 2222 } + enable_reuse_port: false + filter_chains: + - filters: [] + name: foo + tcp_keepalive: + keepalive_probes: 3 + keepalive_time: 4 + keepalive_interval: 5 + )EOF"); + + // Second address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 4, + ListenerComponentFactory::BindType::NoReusePort); + + // First address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 4, + ListenerComponentFactory::BindType::NoReusePort); + + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_SO_KEEPALIVE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_SO_KEEPALIVE.option(), + /* expected_value */ 1, + /* expected_num_calls */ 2); + + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPCNT.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPCNT.option(), + /* expected_value */ 3, + /* expected_num_calls */ 2); + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPIDLE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPIDLE.option(), + /* expected_value */ 4, + /* expected_num_calls */ 2); + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPINTVL.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPINTVL.option(), + /* expected_value */ 5, + /* expected_num_calls */ 2); + addOrUpdateListener(listener); + EXPECT_EQ(1U, manager_->listeners().size()); +} + +TEST_P(ListenerManagerImplWithRealFiltersTest, ListenerKeepaliveOnAdditionalAddressEnabled) { + if (!ENVOY_SOCKET_SO_KEEPALIVE.hasValue()) { + return; // Keepalive is not supported on this platform. + } + + const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( + name: SockoptsListener + address: + socket_address: { address: 127.0.0.1, port_value: 1111 } + additional_addresses: + - address: + socket_address: { address: 127.0.0.1, port_value: 2222 } + tcp_keepalive_override: + tcp_keepalive: + keepalive_probes: 3 + keepalive_time: 4 + keepalive_interval: 5 + enable_reuse_port: false + filter_chains: + - filters: [] + name: foo + tcp_keepalive: {} + )EOF"); + + // Second address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 4, + ListenerComponentFactory::BindType::NoReusePort); + + // First address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 1, + ListenerComponentFactory::BindType::NoReusePort); + + // First & Second address option. + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_SO_KEEPALIVE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_SO_KEEPALIVE.option(), + /* expected_value */ 1, + /* expected_num_calls */ 2); + + // Second address options. + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPCNT.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPCNT.option(), + /* expected_value */ 3); + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPIDLE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPIDLE.option(), + /* expected_value */ 4); + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_TCP_KEEPINTVL.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_TCP_KEEPINTVL.option(), + /* expected_value */ 5); + addOrUpdateListener(listener); + EXPECT_EQ(1U, manager_->listeners().size()); +} + +TEST_P(ListenerManagerImplWithRealFiltersTest, ListenerKeepaliveAdditionalAddressOverrideDisable) { + if (!ENVOY_SOCKET_SO_KEEPALIVE.hasValue()) { + return; // Keepalive is not supported on this platform. + } + + const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( + name: SockoptsListener + address: + socket_address: { address: 127.0.0.1, port_value: 1111 } + additional_addresses: + - address: + socket_address: { address: 127.0.0.1, port_value: 2222 } + tcp_keepalive_override: {} + enable_reuse_port: false + filter_chains: + - filters: [] + name: foo + tcp_keepalive: {} + )EOF"); + + // Second address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 0, + ListenerComponentFactory::BindType::NoReusePort); + + // First address. + expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, + /* expected_num_options */ 1, + ListenerComponentFactory::BindType::NoReusePort); + + // First & Second address option. + expectSetsockopt(/* expected_sockopt_level */ ENVOY_SOCKET_SO_KEEPALIVE.level(), + /* expected_sockopt_name */ ENVOY_SOCKET_SO_KEEPALIVE.option(), + /* expected_value */ 1, + /* expected_num_calls */ 1); + + addOrUpdateListener(listener); + EXPECT_EQ(1U, manager_->listeners().size()); +} + TEST_P(ListenerManagerImplWithRealFiltersTest, LiteralSockoptListenerEnabledWithMultiAddressesNoOverrideOpts) { const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( diff --git a/test/integration/listener_lds_integration_test.cc b/test/integration/listener_lds_integration_test.cc index 51a806d09ef2..96bfa9d0eb0c 100644 --- a/test/integration/listener_lds_integration_test.cc +++ b/test/integration/listener_lds_integration_test.cc @@ -567,6 +567,97 @@ TEST_P(ListenerMultiAddressesIntegrationTest, BasicSuccessWithMultiAddressesAndS } #endif +#ifdef __linux__ +TEST_P(ListenerMultiAddressesIntegrationTest, BasicSuccessWithMultiAddressesAndKeepalive) { + if (!ENVOY_SOCKET_SO_KEEPALIVE.hasValue()) { + return; // Keepalive is not supported on this platform. + } + + on_server_init_function_ = [&]() { + createLdsStream(); + listener_config_.mutable_tcp_keepalive(); + listener_config_.mutable_additional_addresses(0) + ->mutable_tcp_keepalive_override() + ->mutable_tcp_keepalive() + ->mutable_keepalive_probes() + ->set_value(3); + sendLdsResponse({MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + createRdsStream(route_table_name_); + }; + initialize(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + // testing-listener-0 is not initialized as we haven't pushed any RDS yet. + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + // Workers not started, the LDS added listener 0 is in active_listeners_ list. + EXPECT_EQ(test_server_->server().listenerManager().listeners().size(), 1); + registerTestServerPorts({"address1", "address2"}); + + constexpr absl::string_view route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} +)EOF"; + sendRdsResponse(fmt::format(route_config_tmpl, route_table_name_, "cluster_0"), "1"); + test_server_->waitForCounterGe( + fmt::format("http.config_test.rds.{}.update_success", route_table_name_), 1); + // Now testing-listener-0 finishes initialization, Server initManager will be ready. + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initialized); + + test_server_->waitUntilListenersReady(); + // NOTE: The line above doesn't tell you if listener is up and listening. + test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + // Request is sent to cluster_0. + + int response_size = 800; + int request_size = 10; + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, + {"server_id", "cluster_0, backend_0"}}; + + codec_client_ = makeHttpConnection(lookupPort("address1")); + auto response = sendRequestAndWaitForResponse( + Http::TestResponseHeaderMapImpl{ + {":method", "GET"}, {":path", "/"}, {":authority", "host"}, {":scheme", "http"}}, + request_size, response_headers, response_size, /*cluster_0*/ 0); + verifyResponse(std::move(response), "200", response_headers, std::string(response_size, 'a')); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(request_size, upstream_request_->bodyLength()); + codec_client_->close(); + // Wait for the client to be disconnected. + ASSERT_TRUE(codec_client_->waitForDisconnect()); + + codec_client_ = makeHttpConnection(lookupPort("address2")); + auto response2 = sendRequestAndWaitForResponse( + Http::TestResponseHeaderMapImpl{ + {":method", "GET"}, {":path", "/"}, {":authority", "host"}, {":scheme", "http"}}, + request_size, response_headers, response_size, /*cluster_0*/ 0); + verifyResponse(std::move(response2), "200", response_headers, std::string(response_size, 'a')); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(request_size, upstream_request_->bodyLength()); + + int opt_value = 0; + socklen_t opt_len = sizeof(opt_value); + // Verify first address. + EXPECT_TRUE(getSocketOption("testing-listener-0", ENVOY_SOCKET_SO_KEEPALIVE.level(), + ENVOY_SOCKET_SO_KEEPALIVE.option(), &opt_value, &opt_len, 0)); + EXPECT_EQ(opt_len, sizeof(opt_value)); + EXPECT_EQ(1, opt_value); + // Verify second address. + EXPECT_TRUE(getSocketOption("testing-listener-0", ENVOY_SOCKET_SO_KEEPALIVE.level(), + ENVOY_SOCKET_SO_KEEPALIVE.option(), &opt_value, &opt_len, 1)); + EXPECT_EQ(opt_len, sizeof(opt_value)); + EXPECT_EQ(1, opt_value); + + EXPECT_TRUE(getSocketOption("testing-listener-0", ENVOY_SOCKET_TCP_KEEPCNT.level(), + ENVOY_SOCKET_TCP_KEEPCNT.option(), &opt_value, &opt_len, 1)); + EXPECT_EQ(opt_len, sizeof(opt_value)); + EXPECT_EQ(3, opt_value); +} +#endif + #ifdef __linux__ TEST_P(ListenerMultiAddressesIntegrationTest, BasicSuccessWithSocketOptionsOnlySetOnAdditionalAddress) {