Skip to content

Commit 4a3b372

Browse files
charts/cannon: Bundle nginz and expose directly to load balancer (#2421)
By default, incoming network traffic for websockets comes through these network hops: Internet -> LoadBalancer -> kube-proxy -> nginx-ingress-controller -> nginz -> cannon In order to have graceful draining of websockets when something gets restarted (as implemented in #2416 ), as it is not easily possible to implement the graceful draining on nginx-ingress-controller or nginz by itself, with this PR there is now a configuration option to get the following network hops: Internet -> separate LoadBalancer for cannon only -> kube-proxy -> [nginz->cannon (2 containers in the same pod)] More context: https://wearezeta.atlassian.net/wiki/spaces/PS/pages/585564424/How+to+gracefully+drain+cannon+but+not+so+slowly FUTUREWORK: this introduces some nginz config duplication; some way to refactor this (e.g. by moving charts/{cannon, nginz}/* to charts/wire-server/ in a backwards-compatible way) would allow to reduce this duplication. Co-authored-by: jschaul <[email protected]>
1 parent 6b0bdf0 commit 4a3b372

23 files changed

+695
-17
lines changed

.dockerignore

+3
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,6 @@
99
.stack-root-buildah
1010
.local
1111
services/nginz/src/objs
12+
dist-newstyle
13+
.env
14+
.direnv

changelog.d/2-features/cannon-nginz

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Optionally allow to run cannon with its own nginz inside the same pod; and connect to a load balancer directly.
2+
This allows the cannon-slow-drain behaviour implemented in #2416 to take effect by not having other intermediate network hops which could break websocket connections all at once.
3+
Some (internal) context: https://wearezeta.atlassian.net/wiki/spaces/PS/pages/585564424/How+to+gracefully+drain+cannon+but+not+so+slowly
4+
For details on how to configure this, see docs/src/how-to/install/configuration-options.rst

charts/cannon/conf/static/zauth.acl

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
a (blacklist (path "/provider")
2+
(path "/provider/**")
3+
(path "/bot")
4+
(path "/bot/**")
5+
(path "/i/**"))
6+
7+
b (whitelist (path "/bot")
8+
(path "/bot/**"))
9+
10+
p (whitelist (path "/provider")
11+
(path "/provider/**"))
12+
13+
# LegalHold Access Tokens
14+
la (whitelist (path "/notifications")
15+
(path "/assets/v3/**")
16+
(path "/users")
17+
(path "/users/**"))
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,342 @@
1+
{{- define "cannon_nginz_nginx.conf" }}
2+
user {{ .Values.nginx_conf.user }} {{ .Values.nginx_conf.group }};
3+
worker_processes {{ .Values.nginx_conf.worker_processes }};
4+
worker_rlimit_nofile {{ .Values.nginx_conf.worker_rlimit_nofile | default 1024 }};
5+
pid /var/run/nginz.pid;
6+
7+
# nb. start up errors (eg. misconfiguration) may still end up in
8+
# /var/log/nginz/error.log
9+
error_log stderr warn;
10+
11+
events {
12+
worker_connections {{ .Values.nginx_conf.worker_connections | default 1024 }};
13+
multi_accept off;
14+
use epoll;
15+
}
16+
17+
http {
18+
#
19+
# Sockets
20+
#
21+
22+
sendfile on;
23+
tcp_nopush on;
24+
tcp_nodelay on;
25+
26+
#
27+
# Timeouts
28+
#
29+
30+
client_body_timeout 60;
31+
client_header_timeout 60;
32+
keepalive_timeout 75;
33+
send_timeout 60;
34+
35+
ignore_invalid_headers off;
36+
37+
types_hash_max_size 2048;
38+
39+
server_names_hash_bucket_size 64;
40+
server_name_in_redirect off;
41+
42+
large_client_header_buffers 4 8k;
43+
44+
45+
#
46+
# Security
47+
#
48+
49+
server_tokens off;
50+
51+
#
52+
# Logging
53+
#
54+
# Note sanitized_request:
55+
# We allow passing access_token as query parameter for e.g. websockets
56+
# However we do not want to log access tokens.
57+
#
58+
59+
log_format custom_zeta '$remote_addr $remote_user "$time_local" "$sanitized_request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $http_x_forwarded_for $connection $request_time $upstream_response_time $upstream_cache_status $zauth_user $zauth_connection $request_id $proxy_protocol_addr "$http_tracestate"';
60+
access_log /dev/stdout custom_zeta;
61+
62+
#
63+
# Monitoring
64+
#
65+
vhost_traffic_status_zone;
66+
67+
#
68+
# Gzip
69+
#
70+
71+
gzip on;
72+
gzip_disable msie6;
73+
gzip_vary on;
74+
gzip_proxied any;
75+
gzip_comp_level 6;
76+
gzip_buffers 16 8k;
77+
gzip_http_version 1.1;
78+
gzip_min_length 1024;
79+
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
80+
81+
#
82+
# This directive ensures that X-Forwarded-For is used
83+
# as the client's real IP address (since nginz is always
84+
# behind an ELB, remote_addr now becomes the client's real
85+
# IP address)
86+
#
87+
88+
real_ip_header X-Forwarded-For;
89+
set_real_ip_from 0.0.0.0/0;
90+
91+
#
92+
# Rate Limiting Exemptions
93+
#
94+
95+
geo $rate_limit {
96+
default 1;
97+
98+
# IPs to exempt can be added in the .Values.nginx_conf.rate_limit and .Values.nginx_conf.simulators helm values
99+
{{ if (hasKey .Values.nginx_conf "rate_limit_exemptions") }}
100+
{{ range $ip := .Values.nginx_conf.rate_limit_exemptions }}
101+
{{ $ip }} 0;
102+
{{ end }}
103+
{{ end }}
104+
105+
{{ if (hasKey .Values.nginx_conf "simulators") }}
106+
{{ range $ip := .Values.nginx_conf.simulators }}
107+
{{ $ip }} 0;
108+
{{ end }}
109+
{{ end }}
110+
}
111+
112+
#
113+
# Rate Limiting Mapping
114+
#
115+
116+
map $rate_limit $rate_limited_by_addr {
117+
1 "$binary_remote_addr$uri";
118+
0 "";
119+
}
120+
121+
map $rate_limit $rate_limited_by_zuser {
122+
1 $zauth_user;
123+
0 "";
124+
}
125+
126+
map $http_origin $cors_header {
127+
default "";
128+
{{ range $origin := .Values.nginx_conf.allowlisted_origins }}
129+
"https://{{ $origin }}.{{ $.Values.nginx_conf.external_env_domain}}" "$http_origin";
130+
{{ end }}
131+
132+
# Allow additional origins at random ports. This is useful for testing with an HTTP proxy.
133+
# It should not be used in production.
134+
{{ range $origin := .Values.nginx_conf.randomport_allowlisted_origins }}
135+
"~^https://{{ $origin }}.{{ $.Values.nginx_conf.external_env_domain}}(:[0-9]{2,5})?$" "$http_origin";
136+
{{ end }}
137+
}
138+
139+
140+
#
141+
# Rate Limiting
142+
#
143+
144+
limit_req_zone $rate_limited_by_zuser zone=reqs_per_user:12m rate=10r/s;
145+
limit_req_zone $rate_limited_by_addr zone=reqs_per_addr:12m rate=5r/m;
146+
147+
limit_conn_zone $rate_limited_by_zuser zone=conns_per_user:10m;
148+
limit_conn_zone $rate_limited_by_addr zone=conns_per_addr:10m;
149+
150+
# Too Many Requests (420) is returned on throttling
151+
# TODO: Change to 429 once all clients support this
152+
limit_req_status 420;
153+
limit_conn_status 420;
154+
155+
limit_req_log_level warn;
156+
limit_conn_log_level warn;
157+
158+
# Limit by $zauth_user if present and not part of rate limit exemptions
159+
limit_req zone=reqs_per_user burst=20;
160+
limit_conn conns_per_user 25;
161+
162+
#
163+
# Proxied Upstream Services
164+
#
165+
166+
upstream cannon {
167+
least_conn;
168+
keepalive 32;
169+
server localhost:{{ .Values.service.internalPort }};
170+
}
171+
172+
#
173+
# Mapping for websocket connections
174+
#
175+
176+
map $http_upgrade $connection_upgrade {
177+
websocket upgrade;
178+
default '';
179+
}
180+
181+
182+
183+
#
184+
# Locations
185+
#
186+
187+
server {
188+
listen {{ .Values.service.nginz.internalPort }} ssl;
189+
190+
ssl_certificate /etc/wire/nginz/tls/tls.crt;
191+
ssl_certificate_key /etc/wire/nginz/tls/tls.key;
192+
193+
ssl_protocols {{ .Values.nginx_conf.tls.protocols }};
194+
ssl_ciphers {{ .Values.nginx_conf.tls.ciphers }};
195+
196+
# Disable session resumption. See comments in SQPIT-226 for more context and
197+
# discussion.
198+
ssl_session_tickets off;
199+
ssl_session_cache off;
200+
201+
zauth_keystore {{ .Values.nginx_conf.zauth_keystore }};
202+
zauth_acl {{ .Values.nginx_conf.zauth_acl }};
203+
204+
location /status {
205+
zauth off;
206+
access_log off;
207+
208+
return 200;
209+
}
210+
211+
location /vts {
212+
zauth off;
213+
access_log off;
214+
allow 10.0.0.0/8;
215+
allow 127.0.0.1;
216+
deny all;
217+
218+
# Requests with an X-Forwarded-For header will have the real client
219+
# source IP address set correctly, due to the real_ip_header directive
220+
# in the top-level configuration. However, this will not set the client
221+
# IP correctly for clients which are connected via a load balancer which
222+
# uses the PROXY protocol.
223+
#
224+
# Hence, for safety, we deny access to the vts metrics endpoints to
225+
# clients which are connected via PROXY protocol.
226+
if ($proxy_protocol_addr != "") {
227+
return 403;
228+
}
229+
230+
vhost_traffic_status_display;
231+
vhost_traffic_status_display_format html;
232+
}
233+
234+
# Block "Franz" -- http://meetfranz.com
235+
if ($http_user_agent ~* Franz) {
236+
return 403;
237+
}
238+
239+
{{ range $path := .Values.nginx_conf.disabled_paths }}
240+
location ~* ^(/v[0-9]+)?{{ $path }} {
241+
242+
return 404;
243+
}
244+
{{ end }}
245+
246+
#
247+
# Service Routing
248+
#
249+
250+
{{ range $name, $locations := .Values.nginx_conf.upstreams -}}
251+
{{- range $location := $locations -}}
252+
{{- if hasKey $location "envs" -}}
253+
{{- range $env := $location.envs -}}
254+
{{- if or (eq $env $.Values.nginx_conf.env) (eq $env "all") -}}
255+
256+
{{- if $location.strip_version }}
257+
258+
rewrite ^/v[0-9]+({{ $location.path }}) $1;
259+
{{- end }}
260+
261+
{{- $versioned := ternary $location.versioned true (hasKey $location "versioned") -}}
262+
{{- $path := printf "%s%s" (ternary "(/v[0-9]+)?" "" $versioned) $location.path }}
263+
264+
location ~* ^{{ $path }} {
265+
266+
# remove access_token from logs, see 'Note sanitized_request' above.
267+
set $sanitized_request $request;
268+
if ($sanitized_request ~ (.*)access_token=[^&\s]*(.*)) {
269+
set $sanitized_request $1access_token=****$2;
270+
}
271+
272+
{{- if ($location.disable_zauth) }}
273+
zauth off;
274+
275+
# If zauth is off, limit by remote address if not part of limit exemptions
276+
{{- if ($location.unlimited_requests_endpoint) }}
277+
# Note that this endpoint has no rate limit
278+
{{- else -}}
279+
limit_req zone=reqs_per_addr burst=5 nodelay;
280+
limit_conn conns_per_addr 20;
281+
{{- end -}}
282+
{{- end }}
283+
284+
if ($request_method = 'OPTIONS') {
285+
add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
286+
add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
287+
add_header 'Content-Type' 'text/plain; charset=UTF-8';
288+
add_header 'Content-Length' 0;
289+
return 204;
290+
}
291+
292+
proxy_pass http://{{ $name }};
293+
proxy_http_version 1.1;
294+
295+
{{- if ($location.disable_request_buffering) }}
296+
proxy_request_buffering off;
297+
{{ end -}}
298+
{{- if (hasKey $location "body_buffer_size") }}
299+
client_body_buffer_size {{ $location.body_buffer_size -}};
300+
{{- end }}
301+
client_max_body_size {{ $location.max_body_size | default "64k" }};
302+
303+
{{ if ($location.use_websockets) }}
304+
proxy_set_header Upgrade $http_upgrade;
305+
proxy_set_header Connection $connection_upgrade;
306+
proxy_read_timeout 1h;
307+
{{- else }}
308+
proxy_set_header Connection "";
309+
{{ end -}}
310+
311+
{{- if not ($location.disable_zauth) }}
312+
proxy_set_header Authorization "";
313+
{{- end }}
314+
315+
proxy_set_header Z-Type $zauth_type;
316+
proxy_set_header Z-User $zauth_user;
317+
proxy_set_header Z-Connection $zauth_connection;
318+
proxy_set_header Z-Provider $zauth_provider;
319+
proxy_set_header Z-Bot $zauth_bot;
320+
proxy_set_header Z-Conversation $zauth_conversation;
321+
proxy_set_header Request-Id $request_id;
322+
323+
{{- if ($location.allow_credentials) }}
324+
more_set_headers 'Access-Control-Allow-Credentials: true';
325+
{{ end -}}
326+
327+
more_set_headers 'Access-Control-Allow-Origin: $cors_header';
328+
329+
more_set_headers 'Access-Control-Expose-Headers: Request-Id, Location';
330+
more_set_headers 'Request-Id: $request_id';
331+
more_set_headers 'Strict-Transport-Security: max-age=31536000; preload';
332+
}
333+
334+
{{- end -}}
335+
{{- end -}}
336+
337+
{{- end -}}
338+
{{- end -}}
339+
{{- end }}
340+
}
341+
}
342+
{{- end }}

charts/cannon/templates/configmap.yaml

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
apiVersion: v1
22
data:
33
cannon.yaml: |
4-
logNetStrings: True # log using netstrings encoding:
5-
# http://cr.yp.to/proto/netstrings.txt
4+
logFormat: StructuredJSON
65
logLevel: {{ .Values.config.logLevel }}
76
87
cannon:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{{- if and .Values.service.nginz.enabled (not .Values.service.nginz.certManager.enabled ) }}
2+
apiVersion: v1
3+
kind: Secret
4+
metadata:
5+
name: {{ .Values.service.nginz.tls.secretName }}
6+
labels:
7+
wireService: cannon-nginz
8+
app: cannon-nginz
9+
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
10+
release: "{{ .Release.Name }}"
11+
heritage: "{{ .Release.Service }}"
12+
type: kubernetes.io/tls
13+
data:
14+
tls.crt: {{ .Values.secrets.nginz.tls.crt }}
15+
tls.key: {{ .Values.secrets.nginz.tls.key }}
16+
{{- end }}

0 commit comments

Comments
 (0)