Commit 4f168972 authored by Łukasz Nowak's avatar Łukasz Nowak Committed by Łukasz Nowak

caddy-frontend: Fix random 502 EOFs by adding try_duration

try_duration and try_interval are Caddy proxy's switches which allow to deal
with non working backend (https://caddyserver.com/docs/proxy)

The non working backend is the one, to which connection is lost or was not
possible to make, without sending any data.

The default try_duration=5s and try_interval=250ms are chosen, so that in
normal network conditions (with all possible problems in the network, like
lost packets) the browser will have to wait up to 5 seconds to be informed
that backend is inaccessible or for the request to start being processed,
but only a bit more than 250ms if Caddy would have to reestablish connection
to faulty backend.

In order to check it out it is advisable to setup a system, with real backend,
like apache one, and configure iptables to randomly reject packets to it:

  iptables -A INPUT -m statistic --mode random -p tcp --dport <backend_port> \
  --probability 0.05 -j REJECT --reject-with tcp-reset

Using ab or any other tool will results with lot of 502 EOF in the Caddy error
log and also reported by ab. With this configuration there are no more
errors visible to the client, which come from the problems on the network
between Caddy and the backend.
parent ee87b773
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = de7e30546a952e306e2a74f8492ab419 md5sum = 2747f9125c8dffa0c27b79a6902a55cb
[template-common] [template-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
...@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b ...@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend] [template-apache-frontend]
filename = instance-apache-frontend.cfg.in filename = instance-apache-frontend.cfg.in
md5sum = 2c8e1dd8df4a225b4a8d8f70688a85ca md5sum = da7b5804d50c727412885e8b818ca433
[template-apache-replicate] [template-apache-replicate]
filename = instance-apache-replicate.cfg.in filename = instance-apache-replicate.cfg.in
...@@ -30,7 +30,7 @@ md5sum = 6a86edb96b171fbd0a59d0adc9cc906b ...@@ -30,7 +30,7 @@ md5sum = 6a86edb96b171fbd0a59d0adc9cc906b
[template-slave-list] [template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in filename = templates/apache-custom-slave-list.cfg.in
md5sum = 4b06918875e889f850c5aca0d8c57796 md5sum = d476a5d21d448bcad0ab48494b229e28
[template-slave-configuration] [template-slave-configuration]
filename = templates/custom-virtualhost.conf.in filename = templates/custom-virtualhost.conf.in
...@@ -58,11 +58,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b ...@@ -58,11 +58,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost] [template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in filename = templates/default-virtualhost.conf.in
md5sum = a0b5a3dbf7b1d6622a52173ef5a90e72 md5sum = 26e634a37dd6ab17c75207d4fbbaa65a
[template-cached-slave-virtualhost] [template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in filename = templates/cached-virtualhost.conf.in
md5sum = c64f8ac7ec439460877ce5a5c5ccf1f7 md5sum = 907372828d1ceb05c41240078196f439
[template-log-access] [template-log-access]
filename = templates/template-log-access.conf.in filename = templates/template-log-access.conf.in
...@@ -94,7 +94,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5 ...@@ -94,7 +94,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5
[template-nginx-notebook-slave-virtualhost] [template-nginx-notebook-slave-virtualhost]
filename = templates/nginx-notebook-slave.conf.in filename = templates/nginx-notebook-slave.conf.in
md5sum = 2b765db72191197122554df17ad471d1 md5sum = ee3b5c23f1c81aa43ce7cd8f8e327f70
[template-apache-lazy-script-call] [template-apache-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in filename = templates/apache-lazy-script-call.sh.in
......
...@@ -202,6 +202,8 @@ extra-context = ...@@ -202,6 +202,8 @@ extra-context =
key login_ca_crt ca-custom-frontend:rendered key login_ca_crt ca-custom-frontend:rendered
key enable_http2_by_default configuration:enable-http2-by-default key enable_http2_by_default configuration:enable-http2-by-default
key global_disable_http2 configuration:global-disable-http2 key global_disable_http2 configuration:global-disable-http2
key proxy_try_duration configuration:proxy-try-duration
key proxy_try_interval configuration:proxy-try-interval
key access_log caddy-configuration:access-log key access_log caddy-configuration:access-log
key error_log caddy-configuration:error-log key error_log caddy-configuration:error-log
key sixtunnel_executable :sixtunnel_executable key sixtunnel_executable :sixtunnel_executable
......
...@@ -86,6 +86,18 @@ ...@@ -86,6 +86,18 @@
], ],
"title": "Enable QUIC", "title": "Enable QUIC",
"type": "string" "type": "string"
},
"proxy-try-duration": {
"default": 5,
"description": "A time during which Caddy will try to establish connection with a backend. Setting it to 0 will result with immediate return of 502 EOF error to the browser, even if it would be possible to (re)connect to the backend during few moments. More info in https://caddyserver.com/docs/proxy try_durtion.",
"title": "Duration in seconds of trying a backend",
"type": "integer"
},
"proxy-try-interval": {
"default": 250,
"description": "How often Caddy will try to establish connection with a backend during proxy-try-duration. More info in https://caddyserver.com/docs/proxy try_interval",
"title": "Interval in milliseconds of tries during proxy-try-duration",
"type": "integer"
} }
}, },
"title": "Input Parameters", "title": "Input Parameters",
......
...@@ -102,3 +102,5 @@ configuration.enable-quic = false ...@@ -102,3 +102,5 @@ configuration.enable-quic = false
configuration.mpm-graceful-shutdown-timeout = 5 configuration.mpm-graceful-shutdown-timeout = 5
configuration.monitor-httpd-port = 8072 configuration.monitor-httpd-port = 8072
configuration.frontend-name = configuration.frontend-name =
configuration.proxy-try-duration = 5
configuration.proxy-try-interval = 250
...@@ -66,6 +66,10 @@ crl = {{ custom_ssl_directory }}/crl/ ...@@ -66,6 +66,10 @@ crl = {{ custom_ssl_directory }}/crl/
{% do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %} {% do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
{% do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %} {% do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
{# Pass proxy_try_duration and proxy_try_interval #}
{% do slave_instance.__setitem__('proxy_try_duration', proxy_try_duration) %}
{% do slave_instance.__setitem__('proxy_try_interval', proxy_try_interval) %}
{# Set Up log files #} {# Set Up log files #}
{% do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %} {% do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
{% do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %} {% do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
......
...@@ -21,6 +21,8 @@ ...@@ -21,6 +21,8 @@
gzip gzip
# Rewrite part # Rewrite part
proxy / {{ slave_parameter.get('backend_url', '') }} { proxy / {{ slave_parameter.get('backend_url', '') }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
# As backend is trusting REMOTE_USER header unset it always # As backend is trusting REMOTE_USER header unset it always
header_upstream -REMOTE_USER header_upstream -REMOTE_USER
...@@ -42,6 +44,8 @@ ...@@ -42,6 +44,8 @@
# Compress the output # Compress the output
gzip gzip
proxy / {{ slave_parameter.get('https_backend_url', '') }} { proxy / {{ slave_parameter.get('https_backend_url', '') }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
# As backend is trusting REMOTE_USER header unset it always # As backend is trusting REMOTE_USER header unset it always
header_upstream -REMOTE_USER header_upstream -REMOTE_USER
transparent transparent
......
...@@ -67,6 +67,8 @@ ...@@ -67,6 +67,8 @@
{%- for (proxy_name, proxy_comment) in proxy_append_list %} {%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }} # {{ proxy_comment }}
proxy /{{ proxy_name }} {{ backend_url }} { proxy /{{ proxy_name }} {{ backend_url }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
{%- if proxy_name == 'prefer-gzip' %} {%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip without /prefer-gzip
header_upstream Accept-Encoding gzip header_upstream Accept-Encoding gzip
...@@ -138,6 +140,8 @@ ...@@ -138,6 +140,8 @@
{%- for (proxy_name, proxy_comment) in proxy_append_list %} {%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }} # {{ proxy_comment }}
proxy /{{ proxy_name }} {{ backend_url }} { proxy /{{ proxy_name }} {{ backend_url }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
{%- if proxy_name == 'prefer-gzip' %} {%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip without /prefer-gzip
header_upstream Accept-Encoding gzip header_upstream Accept-Encoding gzip
...@@ -216,6 +220,8 @@ ...@@ -216,6 +220,8 @@
{%- for (proxy_name, proxy_comment) in proxy_append_list %} {%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }} # {{ proxy_comment }}
proxy /{{ proxy_name }} {{ backend_url }} { proxy /{{ proxy_name }} {{ backend_url }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
{%- if proxy_name == 'prefer-gzip' %} {%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip without /prefer-gzip
header_upstream Accept-Encoding gzip header_upstream Accept-Encoding gzip
...@@ -281,6 +287,8 @@ ...@@ -281,6 +287,8 @@
{%- for (proxy_name, proxy_comment) in proxy_append_list %} {%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }} # {{ proxy_comment }}
proxy /{{ proxy_name }} {{ slave_parameter.get('url', '') }} { proxy /{{ proxy_name }} {{ slave_parameter.get('url', '') }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
{%- if proxy_name == 'prefer-gzip' %} {%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip without /prefer-gzip
header_upstream Accept-Encoding gzip header_upstream Accept-Encoding gzip
......
...@@ -17,6 +17,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt ...@@ -17,6 +17,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
} }
proxy / {{ https_upstream }} { proxy / {{ https_upstream }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
transparent transparent
insecure_skip_verify insecure_skip_verify
} }
...@@ -25,6 +27,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt ...@@ -25,6 +27,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
to /proxy/{1} to /proxy/{1}
} }
proxy /proxy/ {{ https_upstream }} { proxy /proxy/ {{ https_upstream }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
header_upstream X-Real-IP {remote} header_upstream X-Real-IP {remote}
header_upstream Host {host} header_upstream Host {host}
websocket websocket
...@@ -42,6 +46,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http ...@@ -42,6 +46,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
errors {{ slave_parameter.get('error_log') }} errors {{ slave_parameter.get('error_log') }}
proxy / {{ upstream }} { proxy / {{ upstream }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
transparent transparent
insecure_skip_verify insecure_skip_verify
} }
...@@ -50,6 +56,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http ...@@ -50,6 +56,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
to /proxy/{1} to /proxy/{1}
} }
proxy /proxy/ {{ upstream }} { proxy /proxy/ {{ upstream }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
try_interval {{ slave_parameter['proxy_try_interval'] }}ms
header_upstream X-Real-IP {remote} header_upstream X-Real-IP {remote}
header_upstream Host {host} header_upstream Host {host}
websocket websocket
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment