Update Release Candidate

component/postfix/buildout.cfg

@@ -20,7 +20,7 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
 configure-command = make
-configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl-1.0:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
+configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
 make-targets = non-interactive-package install_root=${:location}
 environment =
   PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s

software/kvm/buildout.hash.cfg

@@ -15,15 +15,15 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
+md5sum = 399b398a8eabfa6126d2a521dc779f9b
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 31b17b55200ea065cb97085283ef5568
+md5sum = 704b6ac6bf42837bcd8f4582c5a746c0
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 28a00c28a972f42627849b25c2792abb
+md5sum = 80b9b70ba1ccbc09deb8f9cad60f352c
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
-md5sum = bd7e5db872b0dbe7716ec49c3907c401
+md5sum = 3e7ff2ba85762ca47b5f90495d492570
 
 [template-kvm-import-script]
 filename = template/kvm-import.sh.jinja2
@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
 
 [template-kvm-export]
 filename = instance-kvm-export.cfg.jinja2
-md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
+md5sum = c15f7600389b0c641622fcfdc42260d8
 
 [template-kvm-export-script]
 filename = template/kvm-export.sh.jinja2
@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc
 [image-download-config-creator]
 _update_hash_filename_ = template/image-download-config-creator.py
 md5sum = 54261e418ab9860efe73efd514c4d47f
+
+[whitelist-firewall-download-controller]
+_update_hash_filename_ = template/whitelist-firewall-download-controller.py
+md5sum = bc64e29546833817636261d1b28aa6dc
+
+[whitelist-domains-default]
+_update_hash_filename_ = template/whitelist-domains-default
+md5sum = e9d40162ba77472775256637a2617d14

software/kvm/instance-kvm-cluster-input-schema.json

@@ -556,6 +556,12 @@
                   "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
                 }
               ]
+            },
+            "whitelist-domains": {
+              "title": "Whitelist domains",
+              "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
+              "type": "string",
+              "textarea": true
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -107,6 +107,12 @@
                   "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
                 }
               ]
+            },
+            "whitelist-domains": {
+              "title": "Whitelist domains",
+              "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
+              "type": "string",
+              "textarea": true
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -24,7 +24,7 @@ global-ipv6 = {{ ipv6 }}
 
 [slap-parameter]
 {% for k, v in slapparameter_dict.items() -%}
-{{ k }} = {{ v }}
+{{ k }} = {{ dumps(v) }}
 {% endfor -%}
 
 [request-common]
@@ -135,14 +135,13 @@ config-document-host = ${apache-conf:ip}
 config-document-port = ${apache-conf:port}
 config-document-path = ${hash-code:passwd}
 config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
-{%- if 'boot-image-url-list' in kvm_parameter_dict %}
-{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
-config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
-{%- endif %}
-{%- if 'boot-image-url-select' in kvm_parameter_dict %}
-{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
-config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
-{%- endif %}
+{%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
+{#-   play nice - use parameter only if present #}
+{%-   if k in kvm_parameter_dict %}
+{#-     play safe - dumps value #}
+config-{{ k }} = {{ dumps(kvm_parameter_dict[k]) }}
+{%-   endif %}
+{%- endfor %}
 config-type = cluster
 
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}

software/kvm/instance-kvm-export.cfg.jinja2

@@ -5,24 +5,11 @@ extends =
 
 parts +=
   cron-entry-backup
-
-  certificate-authority
-  certificate-authority-service
-  publish-connection-information
-  kvm-vnc-promise
-  kvm-disk-image-corruption-promise
-  websockify-sighandler
-  websockify-sighandler-service
-  novnc-promise
-  cron
-  cron-service
-  frontend-promise
-# monitor parts
-  monitor-base
+  ${instance-kvm-parts:parts}
 
 [slap-parameter]
 {% for k, v in slapparameter_dict.items() -%}
-{{ k }} = {{ v }}
+{{ k }} = {{ dumps(v) }}
 {% endfor -%}
 
 # Create the exporter executable, which is a simple shell script

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -14,7 +14,7 @@ extends =
 
 [slap-parameter]
 {% for k, v in slapparameter_dict.items() -%}
-{{ k }} = {{ v }}
+{{ k }} = {{ dumps(v) }}
 {% endfor -%}
 
 [resilient-publish-connection-parameter]

software/kvm/instance-kvm-input-schema.json

@@ -419,6 +419,12 @@
           "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
         }
       ]
+    },
+    "whitelist-domains": {
+      "title": "Whitelist domains",
+      "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
+      "type": "string",
+      "textarea": true
     }
   }
 }

software/kvm/instance-kvm.cfg.jinja2

+{# Workaround empty parameter passing #}
+{# In case of resilient '' is converted to 'None' string, but with slapproxy '' becomes None #}
+{% for k, v in slapparameter_dict.items() %}
+{%   if v == 'None' or v is none %}
+{%     do slapparameter_dict.__setitem__(k, '') %}
+{%   endif %}
+{% endfor %}
 {% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
 {% set enable_http = str(slapparameter_dict.get('enable-http-server', False)).lower() == 'true' -%}
 {% set use_tap = str(slapparameter_dict.get('use-tap', True)).lower() == 'true' -%}
@@ -9,6 +16,7 @@
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
+{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
 {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
@@ -92,22 +100,14 @@ config-state = empty
 config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
 
 [boot-image-url-select-source-config]
-# generates configuration of the image from the user parameter
-# special "magic" is used, to properly support multiline boot-image-url-select
-# but in the same time correctly generate the configuration file
 recipe = slapos.recipe.template:jinja2
 template = inline:
-{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
 {%- raw %}
-  {%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
-  {%- if boot_image_url_select == 'None' %}
-  {#- That's insane protection, is it 'None' because of type = array? #}
-  {%-   set boot_image_url_select = '' %}
-  {%- endif %}
   {{ boot_image_url_select }}
 {% endraw -%}
+boot-image-url-select = {{ dumps(slapparameter_dict['boot-image-url-select']) }}
 context =
-  section slap_parameter slap-parameter
+  key boot_image_url_select :boot-image-url-select
 rendered = ${directory:etc}/boot-image-url-select.json
 
 [boot-image-url-select-processed-config]
@@ -199,17 +199,14 @@ config-state = empty
 config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
 
 [boot-image-url-list-source-config]
-# generates configuration of the image from the user parameter
-# special "magic" is used, to properly support multiline boot-image-url-list
-# but in the same time correctly generate the configuration file
 recipe = slapos.recipe.template:jinja2
 template = inline:
-{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
 {%- raw %}
-  {{ slap_parameter.get('boot-image-url-list') or '' }}
+  {{ boot_image_url_list }}
 {% endraw -%}
+boot-image-url-list = {{ dumps(slapparameter_dict['boot-image-url-list']) }}
 context =
-  section slap_parameter slap-parameter
+  key boot_image_url_list :boot-image-url-list
 rendered = ${directory:etc}/boot-image-url-list.conf
 
 [boot-image-url-list-processed-config]
@@ -976,7 +973,7 @@ keyboard-layout-language = fr
 {% set key_list =  v.split('\n') -%}
 {{ k }} =
   {{ key_list | join('\n  ') }}
-{% elif k == 'boot-image-url-list' %}
+{% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
 {# needs to decorate possibly multiline or maybe unsafe value #}
 {{ k }} = {{ dumps(v) }}
 {% else -%}
@@ -1042,7 +1039,62 @@ command-line =
 
 {% endif -%}
 
-[buildout]
+{%   do part_list.append('whitelist-firewall') -%}
+[whitelist-firewall]
+recipe = slapos.cookbook:wrapper
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
+command-line =
+  {{ dnsresolver_executable }}
+  --style list
+  --output ${:output}
+  ${:source}
+
+source =
+  ${whitelist-domains-request:rendered}
+  {{ whitelist_domains_default }}
+  ${whitelist-domains-resolv.conf:output}
+  ${whitelist-domains-download:output}
+output = ${buildout:directory}/.slapos-whitelist-firewall
+
+[whitelist-firewall-directory]
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/whitelist-firewall
+command = mkdir -p ${:location}
+update-command = ${:command}
+
+[whitelist-domains-request]
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{%- raw %}
+  {%- for domain in whitelist_domains.split() %}
+  {{ domain }}
+  {%- endfor %}
+{% endraw -%}
+rendered = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
+whitelist-domains = {{ dumps(whitelist_domains) }}
+extensions = jinja2.ext.do
+context =
+  key whitelist_domains :whitelist-domains
+
+[whitelist-domains-resolv.conf]
+recipe = plone.recipe.command
+output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
+update-command = ${:command}
+command =
+  egrep ^nameserver /etc/resolv.conf  | cut -d ' ' -f 2 > ${:output}
+
+[whitelist-domains-download]
+recipe = slapos.cookbook:wrapper
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
+output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
+interval = 3600
+command-line = {{ python_executable }} {{ whitelist_firewall_download_controller }} {{ curl_executable_location }} 3600 ${:output} ${:url}
+
+url = https://stream.nxdcdn.com/rapidspace-whitelist-domains
+
+[instance-kvm-parts]
 parts =
   certificate-authority
   certificate-authority-service
@@ -1081,6 +1133,9 @@ parts =
 # Complete parts with sections
   {{ part_list | join('\n  ') }}
 
+[buildout]
+parts = ${instance-kvm-parts:parts}
+
 extends =
 #  Add extends list
  {{ extends_list | join('\n  ') }}

software/kvm/instance.cfg.in

@@ -80,9 +80,12 @@ extra-context =
   raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
   raw curl_executable_location ${curl:location}/bin/curl
   raw dash_executable_location ${dash:location}/bin/dash
+  raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
   raw dcron_executable_location ${dcron:location}/sbin/crond
   raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
   raw file_download_script ${file-download-script:location}/${file-download-script:filename}
+  raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
+  raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
   raw image_download_controller ${image-download-controller:target}
   raw image_download_config_creator ${image-download-config-creator:target}
   raw logrotate_cfg ${template-logrotate-base:rendered}

software/kvm/software.cfg

@@ -182,6 +182,10 @@ path = download_file.in
 filename = download_file
 on-update = true
 
+[whitelist-domains-default]
+<= download-template-base
+filename = whitelist-domains-default
+
 [template-httpd]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
@@ -199,6 +203,10 @@ mode = 640
 [image-download-config-creator]
 <= image-download-controller
 
+[whitelist-firewall-download-controller]
+<= image-download-controller
+
+
 [versions]
 websockify = 0.9.0
 

software/kvm/template/whitelist-domains-default

+# Minimal whitelisted domains needed to instantiate the instance
+# Does not guarantee good usage of the guest VM itself
+# The full list shall end up in whitelist-domains-download:url
+# shcache.nxdcdn.com is default source for a lot of operations
+shacache.nxdcdn.com
+# stream.nxdcdn.com is needed by partition itself
+stream.nxdcdn.com
+# partition has to access default SlapOS Master
+slap.vifib.com
+slapos.vifib.com
+# Partition needs access to SlapOS Master related resources
+hnode.cdn.vifib.com
+node.cdn.vifib.com

software/kvm/template/whitelist-firewall-download-controller.py

+#!/usr/bin/env python
+
+import os
+import subprocess
+import sys
+import time
+import logging
+
+
+# Note: Assuring only one running instance is not done, as this script is only
+#       run from supervisord, which does it already
+if __name__ == "__main__":
+  curl, sleep, output, url = sys.argv[1:]
+  sleep = int(sleep)
+  tmp_output = output + '.tmp'
+
+  logging.basicConfig(
+    format='%%(asctime)s [%%(levelname)s] %s : %%(message)s' % (url,),
+    level=logging.DEBUG)
+  logging.info('Redownloading each %is', sleep)
+  while True:
+    logging.info('Fetching')
+    try:
+      subprocess.check_output([
+        curl,
+        '--location',  # follow redirects
+        '--no-progress-meter',  # do not tell too much
+        '--max-time', '600',  # 10 minutes is maximum
+        '--fail',  # fail in case of wrong HTTP code
+        '--output', tmp_output, url],
+        stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+      logging.error('Problem while downloading: %r', e.output.strip())
+    if os.path.exists(tmp_output):
+      logging.info('Stored output')
+      os.rename(tmp_output, output)
+    logging.info('Sleeping for %is', sleep)
+    time.sleep(sleep)

software/re6stnet/software.cfg

@@ -5,6 +5,7 @@ extends =
   ../../component/openssl/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/apache/buildout.cfg
+  ../../component/geoip2/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../stack/slapos.cfg
@@ -36,11 +37,13 @@ eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
   pyOpenSSL
-  re6stnet
+  re6stnet[geoip]
 initialization =
   import os
-  path = os.environ.get('PATH', '')
-  os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
+  path = os.environ.get('PATH', '/bin:/usr/bin')
+  os.environ['PATH'] = '${openssl:location}/bin:${git:location}/bin' + (
+    path and ':' + path)
+  os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
 depends = ${re6stnet-develop:recipe}
 
 [re6stnet-backup]
@@ -59,6 +62,7 @@ template =
   h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
   git update-index --add --cacheinfo 0644 "$h" registry.sql
   git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
+    -c user.name="$${0##*/}" -c user.email="$USER@`hostname`" \
     -c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
   [ ! "$3" ] || git push --mirror "$3"
 context =

stack/slapos.cfg

@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.5
-slapos.toolbox = 0.117
+slapos.toolbox = 0.118
 stevedore = 1.21.0
-subprocess32 = 3.5.3
+subprocess32 = 3.5.4
 unicodecsv = 0.14.1
 wcwidth = 0.2.5
 wheel = 0.35.1