caddy-frontend: Implement type:websocket

By default whole slave makes websocket connection to the backend.
With websocket-path, only the path has websocket style connections,
the rest is standard HTTP.

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = bde0f62dfe2eeef8f10b4315535095cb
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = a4303904fa1dfebcbb40f28cd715e7cf
+md5sum = d62aefe002ec13875924e4c219914795
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 71dfc1c57988416f5a40ced83acda2a7
+md5sum = 75439cb035393e68c73672b224bead54
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -54,7 +54,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 9de6875635038f88be4f039e03deb1c0
+md5sum = 0c5ef7f26a142c3ab53e835d2caa698d
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -82,7 +82,7 @@ context =
 {%   set slave_type = slave.get('type') %}
 {%   if slave_type == 'eventsource' %}
 {%     do slave_error_list.append('type:eventsource is not implemented') %}
-{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook'] %}
+{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
 {#   BBB: apache_custom_https AND apache_custom_http #}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -107,6 +107,22 @@
       "title": "type:zope Backend Path",
       "type": "string"
     },
+    "websocket-path-list": {
+      "default": "",
+      "description": "Space separated list of path to the websocket application. If not set the whole slave will be websocket, if set then / will be HTTP, and /<websocket-path> will be WSS. In order to have ' ' in the space use '%20'",
+      "title": "type:websocket Websocket Application Path List",
+      "type": "string"
+    },
+    "websocket-transparent": {
+      "default": "true",
+      "description": "If set to false, websocket slave will be without Caddy's transparent proxy mode. Depending on the application the setting shall be false or true. Defaults to true for transparent proxying.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "type:websocket Transparent proxy",
+      "type": "string"
+    },
     "prefer-gzip-encoding-to-backend": {
       "default": "false",
       "description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
@@ -154,6 +170,7 @@
         "zope",
         "redirect",
         "notebook",
+        "websocket",
         "eventsource"
       ],
       "title": "Backend Type",

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -248,13 +248,16 @@ rendered = {{ caddy_configuration_directory }}/${:filename}
 
 {%   if caddy_custom_http or caddy_custom_https %}
 template = {{ template_custom_slave_configuration }}
+extra-context =
+    section slave_parameter {{ slave_configuration_section_name }}
 {%   else %}
 template = {{ template_default_slave_configuration }}
+extra-context =
+    section slave_parameter {{ slave_configuration_section_name }}
+    import urllib_module urllib
 {%   endif %}
 
 filename = {{ '%s.conf' % slave_reference }}
-extra-context =
-    section slave_parameter {{ slave_configuration_section_name }}
 {{ '\n' }}
 
 

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -23,8 +23,18 @@
 {%-   do https_host_list.append('https://%s:%s' % (host, slave_parameter['https_port'] )) %}
 {%- endfor %} {#- for host in host_list #}
 {%- set default_path = slave_parameter.get('default-path', '').strip('/') | urlencode %}
-{%- if slave_type == 'notebook' %}
-{# notebook needs http 1.1 max #}
+{%- set websocket_path_list = [] %}
+{%- for websocket_path in slave_parameter.get('websocket-path-list', '').split() %}
+{%-   set websocket_path = websocket_path.strip('/') %}
+{#- Unquote the path, so %20 and similar can be represented correctly #}
+{%-   set websocket_path = urllib_module.unquote(websocket_path.strip()) %}
+{%-   if websocket_path %}
+{%-     do websocket_path_list.append(websocket_path) %}
+{%-   endif %}
+{%- endfor %}
+{%- set websocket_transparent = slave_parameter.get('websocket-transparent', 'true').lower() in TRUE_VALUES %}
+{%- if slave_type in ['notebook', 'websocket'] %}
+{# websocket style needs http 1.1 max #}
 {%-   set enable_h2 = False %}
 {%- endif %}
 
@@ -72,7 +82,7 @@
     if {>Accept-Encoding} not_match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
     to {1}
   }
-{%    elif slave_type != 'notebook' %}
+{%    elif slave_type not in ['notebook', 'websocket'] %}
   rewrite {
     regexp (.*)
     to {1}
@@ -182,6 +192,38 @@
     without /proxy/
     insecure_skip_verify
   }
+{%- elif slave_type == 'websocket' %}
+{%-   if websocket_path_list %}
+  proxy / {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+{%-     if websocket_transparent %}
+    transparent
+{%-     endif %}
+    insecure_skip_verify
+  }
+{%-     for websocket_path in websocket_path_list %}
+  proxy /{{ websocket_path }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    websocket
+{%-       if websocket_transparent %}
+    transparent
+{%-       endif %}
+    insecure_skip_verify
+  }
+{%-     endfor %}
+{%-   else %}
+  proxy / {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    websocket
+{%-   if websocket_transparent %}
+    transparent
+{%-   endif %}
+    insecure_skip_verify
+  }
+{%-   endif %}
 {%- else %} {#- if slave_type ==  'zope' and backend_url #}
   # Default configuration
 {%-   if default_path %}

software/caddy-frontend/test/test.py

@@ -1090,6 +1090,26 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'type': 'notebook',
       },
+      'type-websocket': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+      },
+      'type-websocket-websocket-path-list': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-path-list': '////ws//// /with%20space/',
+      },
+      'type-websocket-websocket-transparent-false': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-transparent': 'false',
+      },
+      'type-websocket-websocket-path-list-websocket-transparent-false': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-path-list': '////ws//// /with%20space/',
+        'websocket-transparent': 'false',
+      },
       'type-eventsource': {
         'url': cls.backend_url,
         'type': 'eventsource',
@@ -1207,9 +1227,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     expected_parameter_dict = {
       'monitor-base-url': None,
       'domain': 'example.com',
-      'accepted-slave-amount': '44',
+      'accepted-slave-amount': '48',
       'rejected-slave-amount': '4',
-      'slave-amount': '48',
+      'slave-amount': '52',
       'kedifa-caucase-url': 'http://[%s]:%s' % (
         SLAPOS_TEST_IPV6, CAUCASE_PORT),
       'rejected-slave-dict': {
@@ -2053,10 +2073,200 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse(
       isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
 
-  @skip('Feature postponed')
   def test_type_websocket(self):
-    # Pure websocket configurable frontend
-    raise NotImplementedError
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+
+  def test_type_websocket_websocket_transparent_false(self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-transparent-false')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+
+  def test_type_websocket_websocket_path_list(self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-path-list')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('connection' in j['Incoming Headers'].keys())
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/ws/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'with%20space/test-path', headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/with%20space/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+  def test_type_websocket_websocket_path_list_websocket_transparent_false(
+    self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-path-list-websocket-transparent-false')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('connection' in j['Incoming Headers'].keys())
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/ws/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'with%20space/test-path', headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/with%20space/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
 
   @skip('Feature postponed')
   def test_type_eventsource(self):

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -72,6 +72,14 @@ T-2/var/log/httpd/_type-notebook_access_log
 T-2/var/log/httpd/_type-notebook_error_log
 T-2/var/log/httpd/_type-redirect_access_log
 T-2/var/log/httpd/_type-redirect_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket_access_log
+T-2/var/log/httpd/_type-websocket_error_log
 T-2/var/log/httpd/_type-zope-default-path_access_log
 T-2/var/log/httpd/_type-zope-default-path_error_log
 T-2/var/log/httpd/_type-zope-path_access_log

software/caddy-frontend/test/test_data/test.TestSlave.test_promise_run_plugin-CADDY.txt

@@ -80,6 +80,14 @@ T-2/etc/plugin/check-_type-notebook-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-notebook-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-error-log-last-day.py: OK

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -72,6 +72,14 @@ T-2/var/log/httpd/_type-notebook_access_log
 T-2/var/log/httpd/_type-notebook_error_log
 T-2/var/log/httpd/_type-redirect_access_log
 T-2/var/log/httpd/_type-redirect_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket_access_log
+T-2/var/log/httpd/_type-websocket_error_log
 T-2/var/log/httpd/_type-zope-default-path_access_log
 T-2/var/log/httpd/_type-zope-default-path_error_log
 T-2/var/log/httpd/_type-zope-path_access_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_promise_run_plugin-CADDY.txt

@@ -80,6 +80,14 @@ T-2/etc/plugin/check-_type-notebook-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-notebook-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-error-log-last-day.py: OK