Update Release Candidate

component/ghostscript/buildout.cfg

@@ -2,17 +2,22 @@
 extends =
   ../fontconfig/buildout.cfg
   ../freetype/buildout.cfg
+  ../libjpeg/buildout.cfg
   ../libtiff/buildout.cfg
   ../libxml2/buildout.cfg
   ../pkgconfig/buildout.cfg
+  ../tesseract/buildout.cfg
   ../xz-utils/buildout.cfg
 
 parts = ghostscript
 
-[ghostscript-common]
+[ghostscript]
 recipe = slapos.recipe.cmmi
 shared = true
-pkg_config_depends = ${libtiff:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}
+url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs9540/ghostscript-9.54.0.tar.gz
+md5sum = 5d571792a8eb826c9f618fb69918d9fc
+pkg_config_depends = ${libtiff:location}/lib/pkgconfig:${libjpeg:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}
+# XXX --with-tessdata work arounds a slaprunner bug of having softwares installed in a path containing //
 configure-options =
   --disable-cups
   --disable-threadsafe
@@ -20,18 +25,18 @@ configure-options =
   --without-libidn
   --without-x
   --with-drivers=FILES
-# it seems that parallel build sometimes fails for ghostscript.
-make-options = -j1
+  --with-tessdata=$(python -c 'print("""${:tessdata-location}""".replace("//", "/"))')
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
-  LDFLAGS=-Wl,-rpath=${fontconfig:location}/lib -Wl,-rpath=${freetype:location}/lib -Wl,-rpath=${libtiff:location}/lib
+  CFLAGS=-I${libjpeg:location}/include
+  LDFLAGS=-Wl,-rpath=${fontconfig:location}/lib -Wl,-rpath=${freetype:location}/lib -Wl,-rpath=${libtiff:location}/lib -L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib
   LD_LIBRARY_PATH=${fontconfig:location}/lib:${freetype:location}/lib:${libtiff:location}/lib:${libxml2:location}/lib
 
-[ghostscript]
-<= ghostscript-9
+# configure gives priority to local jpeg library and refuse mixing local libjpeg with "system" libtiff.
+# We remove this local jpeg library source folder so that configure picks up the slapos versions of these libraries.
+pre-configure = rm -r jpeg
 
-[ghostscript-9]
-<= ghostscript-common
-url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs950/ghostscript-9.50.tar.xz
-md5sum = 6cea6bae4a7cdfac6ccb09f07f0caf8c
+post-make-hook = ${tesseract-download-traineddata:post-make-hook}
+tessdata-location = @@LOCATION@@/share/tessdata/
+tessdata-urls = ${tesseract-download-traineddata:urls}

component/leptonica/buildout.cfg

@@ -7,17 +7,14 @@ extends =
   ../libtiff/buildout.cfg
   ../webp/buildout.cfg
   ../giflib/buildout.cfg
-  ../patch/buildout.cfg
 
 [leptonica]
 recipe = slapos.recipe.cmmi
-url = http://www.leptonica.com/source/leptonica-1.76.0.tar.gz
-md5sum = a263a5e4f7e8f8a661fb121a265d2d20
 shared = true
+url = http://www.leptonica.org/source/leptonica-1.80.0.tar.gz
+md5sum = d640d684234442a84c9e8902f0b3ff36
 configure-options =
   --disable-static
-
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${libjpeg:location}/include -I${libpng:location}/include -I${libtiff:location}/include -I${webp:location}/include -I${giflib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${libpng:location}/lib -Wl,-rpath=${libpng:location}/lib -L${libtiff:location}/lib -Wl,-rpath=${libtiff:location}/lib -L${webp:location}/lib -Wl,-rpath=${webp:location}/lib -L${giflib:location}/lib -Wl,-rpath=${giflib:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s

component/leptonica/leptonica-1.69-zlib-include.patch

-This patch is originally taken from:
-
-http://leptonica.googlecode.com/issues/attachment?aid=560001000&name=zlib-include.patch&token=m2sugSYxB4xwAuNgrKXyHTxBYNg%3A1337345966091
-
-To fix the following issue with leptonica:
-
-http://code.google.com/p/leptonica/issues/detail?id=56
-
-diff -Nurd -x'*~' leptonica-1.68.orig/src/pngio.c leptonica-1.68/src/pngio.c
---- leptonica-1.68.orig/src/pngio.c	2011-02-01 00:41:12.000000000 -0500
-+++ leptonica-1.68/src/pngio.c	2011-07-09 09:17:17.000000000 -0400
-@@ -108,6 +108,10 @@
- 
- #include "png.h"
- 
-+#ifdef HAVE_LIBZ
-+#include "zlib.h"
-+#endif
-+
- /* ----------------Set defaults for read/write options ----------------- */
-     /* strip 16 bpp --> 8 bpp on reading png; default is for stripping */
- static l_int32   var_PNG_STRIP_16_TO_8 = 1;

component/tesseract/buildout.cfg

@@ -10,43 +10,34 @@ extends =
   ../fontconfig/buildout.cfg
   ../lcms/buildout.cfg
   ../pkgconfig/buildout.cfg
-
+  ./buildout.hash.cfg
 parts =
   tesseract
-  tesseract-traineddata
-  tesseract-eng-traineddata
-  tesseract-osd-traineddata
 
 [tesseract]
 recipe = slapos.recipe.cmmi
-url = https://github.com/tesseract-ocr/tesseract/archive/6b250b58121a9858d3e3019a78a6f7d421bd0fc7.tar.gz
-md5sum = fdc38148ad8eb1bd0485a217503dd6d5
+shared = true
+url = https://github.com/tesseract-ocr/tesseract/archive/refs/tags/4.1.1.tar.gz
+md5sum = 51fe2bcbff1bbce77a25d180fd247f7d
 pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
 pre-configure =
   autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability
 
-configure-options =
-  --disable-static
-  --datarootdir=${tesseract-traineddata:location}
-# XXX: tesseract seems not easily configurable at runtime about where to find
-# its trained data, so we set its datarootdir above to a controlled location
-
 environment =
   PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   LDFLAGS=-L${leptonica:location}/lib -Wl,-rpath=${leptonica:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
-[tesseract-traineddata]
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+post-make-hook = ${tesseract-download-traineddata:post-make-hook}
+tessdata-urls = ${tesseract-download-traineddata:urls}
+tessdata-location = @@LOCATION@@/share/tessdata/
 
-[tesseract-eng-traineddata]
-recipe = slapos.recipe.build:download
-destination = ${tesseract-traineddata:location}/tessdata/eng.traineddata
-url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata
-md5sum = 57e0df3d84fed9fbf8c7a8e589f8f012
 
-[tesseract-osd-traineddata]
-recipe = slapos.recipe.build:download
-destination = ${tesseract-traineddata:location}/tessdata/osd.traineddata
-url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
-md5sum = 7611737524efd1ce2dde67eff629bbcf
+[tesseract-download-traineddata]
+post-make-hook = ${:_profile_base_location_}/${download-tessdata.py:filename}#${download-tessdata.py:md5sum}:post_make_hook
+urls =
+    https://raw.githubusercontent.com/tesseract-ocr/tessdata/4.1.0/eng.traineddata#57e0df3d84fed9fbf8c7a8e589f8f012
+    https://raw.githubusercontent.com/tesseract-ocr/tessdata/4.1.0/osd.traineddata#7611737524efd1ce2dde67eff629bbcf
+    https://raw.githubusercontent.com/tesseract-ocr/tessdata/4.1.0/fra.traineddata#a73e70c872f262895d93976febeb1638
+    https://raw.githubusercontent.com/tesseract-ocr/tessdata/4.1.0/jpn.traineddata#af3a30a9bec904e106aa8521e7caaeca
+    https://raw.githubusercontent.com/tesseract-ocr/tessdata/4.1.0/chi_sim.traineddata#6965cb3213edd961cb16264e2ea45f5c

component/tesseract/buildout.hash.cfg

+[download-tessdata.py]
+filename = download-tessdata.py
+md5sum = 2d283a6d8662d6bb8c9de7b26162b702

component/tesseract/download-tessdata.py

+# This is a post-make hook script to download tesseract training data.
+#
+# This script uses the following buildout options:
+#   - tessdata-urls: list of URLs and their expected md5sum as URL fragments
+#   - tessdata-location: path where to install the data.
+
+import zc.buildout
+import os
+
+
+def post_make_hook(options, buildout, env):
+  if not os.path.exists(options['tessdata-location']):
+    os.makedirs(options['tessdata-location'])
+
+  download = zc.buildout.download.Download(
+      buildout['buildout'],
+      hash_name=True,
+  )
+  for url in options['tessdata-urls'].splitlines():
+    url, _, md5sum = url.partition('#')
+    if url:
+      download(
+          url,
+          md5sum=md5sum,
+          path=os.path.join(options['tessdata-location'],
+                            os.path.basename(url)),
+      )

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.184'
+version = '1.0.197'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -40,6 +40,7 @@ extras_require = {
         'jsonschema',
         'mock',
         'testfixtures',
+        'requests',
     ),
 }
 

slapos/recipe/re6stnet/__init__.py

@@ -49,9 +49,8 @@ class Recipe(GenericBaseRecipe):
     self.software_release_url = slap_connection['software-release-url']
     self.key_file = slap_connection.get('key-file')
     self.cert_file = slap_connection.get('cert-file')
-    self.slave_list = json.loads(options['slave-instance-list'])
+    self.slave_list = options['slave-instance-list']
 
-    options['slave-amount'] = '%s' % len(self.slave_list)
     return GenericBaseRecipe.__init__(self, buildout, name, options)
 
   def getSerialFromIpv6(self, ipv6):

slapos/recipe/simplehttpserver/__init__.py

@@ -35,7 +35,7 @@ class Recipe(GenericBaseRecipe):
 
     base_path = options['base-path']
     if options.get('use-hash-url', 'True') in ['true', 'True']:
-      pool = string.letters + string.digits
+      pool = string.ascii_letters + string.digits
       hash_string = ''.join(random.choice(pool) for i in range(64))
       path = os.path.join(base_path, hash_string)
   

slapos/recipe/simplehttpserver/simplehttpserver.py

@@ -43,20 +43,23 @@ class ServerHandler(SimpleHTTPRequestHandler):
     form = cgi.FieldStorage(
       fp=self.rfile,
       headers=self.headers,
-      environ={'REQUEST_METHOD':'POST',
-                'CONTENT_TYPE':self.headers['Content-Type']}
+      environ={'REQUEST_METHOD': 'POST',
+               'CONTENT_TYPE': self.headers['Content-Type']}
     )
-    name = form['path'].value
+    name = form['path'].value.decode('utf-8')
     content = form['content'].value
-    method = 'a'
+    method = 'ab'
     if 'clear' in form and form['clear'].value == '1':
-      method = 'w'
+      method = 'wb'
     self.writeFile(name, content, method)
     self.respond(200, type=self.headers['Content-Type'])
     self.wfile.write(b"Content written to %s" % str2bytes(name))
 
-  def writeFile(self, filename, content, method='a'):
-    file_path = os.path.join(self.document_path, filename)
+  def writeFile(self, filename, content, method='ab'):
+    file_path = os.path.abspath(os.path.join(self.document_path, filename))
+    if not file_path.startswith(self.document_path):
+      self.respond(403, 'text/plain')
+      self.wfile.write(b"Forbidden")
 
     try:
       os.makedirs(os.path.dirname(file_path))

slapos/test/recipe/test_re6stnet.py

@@ -38,7 +38,7 @@ class Re6stnetTest(unittest.TestCase):
                 'drop-service-wrapper': os.path.join(self.base_dir, 'drop_wrapper'),
                 'check-service-wrapper': os.path.join(self.base_dir, 'check_wrapper'),
                 'revoke-service-wrapper': os.path.join(self.base_dir, 'revoke_wrapper'),
-                'slave-instance-list': '{}'
+                'slave-instance-list': [],
                 }
     
   def tearDown(self):
@@ -119,11 +119,10 @@ class Re6stnetTest(unittest.TestCase):
   def test_install(self):
     self.options.update({
         'ipv6-prefix': '2001:db8:24::/48',
-        'slave-instance-list': '''[
+        'slave-instance-list': [
             {"slave_reference":"SOFTINST-58770"},
             {"slave_reference":"SOFTINST-58778"}
             ]
-            '''
         })
 
     recipe = self.new_recipe()
@@ -161,7 +160,7 @@ class Re6stnetTest(unittest.TestCase):
     
     # Remove one element
     self.options.update({
-        "slave-instance-list": """[{"slave_reference":"SOFTINST-58770"}]"""
+        "slave-instance-list": [{"slave_reference":"SOFTINST-58770"}]
         })
     recipe = self.new_recipe()
     recipe.generateCertificate = self.fake_generateCertificates

slapos/test/recipe/test_simplehttpserver.py

+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+import time
+
+from six.moves.urllib import parse as urlparse
+import requests
+
+from slapos.recipe import simplehttpserver
+from slapos.test.utils import makeRecipe
+
+
+class SimpleHTTPServerTest(unittest.TestCase):
+  process = None
+
+  def setUp(self):
+    self.base_path = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.base_path)
+    self.install_dir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.install_dir)
+    self.wrapper = os.path.join(self.install_dir, 'server')
+    host, port = os.environ['SLAPOS_TEST_IPV4'], 9999
+    self.server_url = 'http://{host}:{port}'.format(host=host, port=port)
+    self.recipe = makeRecipe(
+        simplehttpserver.Recipe,
+        options={
+            'base-path': self.base_path,
+            'host': host,
+            'port': port,
+            'log-file': os.path.join(self.install_dir, 'simplehttpserver.log'),
+            'wrapper': self.wrapper,
+        },
+        name='simplehttpserver',
+    )
+
+  def tearDown(self):
+    if self.process:
+      self.process.terminate()
+      self.process.wait()
+
+  def test_options(self):
+    self.assertNotEqual(self.recipe.options['path'], '')
+    self.assertEqual(
+        self.recipe.options['root-dir'],
+        os.path.join(
+            self.base_path,
+            self.recipe.options['path'],
+        ))
+
+  def test_install(self):
+    self.assertEqual(self.recipe.install(), self.wrapper)
+    self.process = subprocess.Popen(
+        self.wrapper,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    server_base_url = urlparse.urljoin(
+        self.server_url,
+        self.recipe.options['path'],
+    )
+    for i in range(16):
+      try:
+        resp = requests.get(server_base_url)
+        break
+      except requests.exceptions.ConnectionError:
+        time.sleep(i * .1)
+    else:
+      self.fail(
+          'server did not start.\nout: %s error: %s' % self.process.communicate())
+    self.assertIn('Directory listing for /', resp.text)
+
+    resp = requests.post(
+        server_base_url,
+        files={
+            'path': 'hello.txt',
+            'content': b'hello',
+        },
+    )
+    self.assertEqual(resp.status_code, requests.codes.ok)
+    with open(
+        os.path.join(self.base_path, self.recipe.options['path'],
+                     'hello.txt')) as f:
+      self.assertEqual(f.read(), 'hello')
+
+    self.assertIn('hello.txt', requests.get(server_base_url).text)
+    self.assertEqual(
+        requests.get(server_base_url + '/hello.txt').text, 'hello')
+
+    # incorrect paths are refused
+    for path in '/hello.txt', '../hello.txt':
+      resp = requests.post(
+          server_base_url,
+          files={
+              'path': path,
+              'content': b'hello',
+          },
+      )
+      self.assertEqual(resp.status_code, requests.codes.forbidden)

software/caddy-frontend/buildout.hash.cfg

@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 1696321871bb71083550c95d77487602
+md5sum = ab0c9ea9bfe7a63a267331d9e010d0c6
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -15,7 +15,6 @@ CONFIG proxy.config.env_prep STRING example_prep.sh
 CONFIG proxy.config.alarm_email STRING nobody
 CONFIG proxy.config.syslog_facility STRING LOG_DAEMON
 CONFIG proxy.config.output.logfile STRING traffic.out
-CONFIG proxy.config.snapshot_dir STRING snapshots
 CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
 CONFIG proxy.config.admin.number_config_bak INT 0
 LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}

software/theia/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 942e269192d284e563a84ae710c0595b
+md5sum = 11274189a4492204e046fef45fdd809c
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = 9a5d6ac099c460481af7c1bdfd8de770
+md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock

software/theia/instance-theia.cfg.jinja.in

@@ -140,7 +140,9 @@ return = domain secure_access
 name = {{ parameter_dict['frontend-name'] }}
 software-url = {{ parameter_dict['frontend-sr'] }}
 software-type = {{ parameter_dict['frontend-sr-type'] }}
+{%- if parameter_dict.get('frontend-guid') %}
 sla-instance_guid = {{ parameter_dict['frontend-guid'] }}
+{%- endif %}
 
 {% if additional_frontend %}
 [remote-additional-frontend]
@@ -148,7 +150,9 @@ sla-instance_guid = {{ parameter_dict['frontend-guid'] }}
 name = {{ parameter_dict['additional-frontend-name'] }}
 software-url = {{ parameter_dict['additional-frontend-sr'] }}
 software-type = {{ parameter_dict['additional-frontend-sr-type'] }}
+{%- if parameter_dict.get('additional-frontend-guid') %}
 sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }}
+{%- endif %}
 {% endif %}
 
 
@@ -281,7 +285,7 @@ install =
   self.buildout['buildout']['offline'] = 'false'
   try:
     gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
-      '''$${slap-configuration:root-instance-title}'''
+      b'''{{ root_title }}'''
     ).hexdigest() + "?s=256&d=retro"
     shutil.copy(self.download(gravatar_url), '''$${:location}''')
   except Exception:
@@ -444,11 +448,13 @@ template =
       $${slapos-standalone-config:ipv6} \
       $${slapos-standalone-config:port} \
       $${slapos-standalone-config:computer-id} \
-      {% if parameter_dict.get('embedded-sr') %} \
+      {%- if parameter_dict.get('embedded-sr') %}
       --sr='{{ parameter_dict['embedded-sr'] }}' \
+      {%- if parameter_dict.get('embedded-sr-type') %}
       --srtype='{{ parameter_dict['embedded-sr-type'] }}' \
+      {%- endif %}
       --srparams='$${embedded-instance-parameters:rendered}' \
-      {% endif %} \
+      {%- endif %}
       $${slap-connection:server-url} \
       $${slap-connection:computer-id} \
       $${slap-connection:partition-id} \

software/theia/instance.cfg.in

@@ -27,6 +27,7 @@ mode = 0644
 context =
   jsonkey default_parameter_dict :default-parameters
   key parameter_dict slap-configuration:configuration
+  key root_title slap-configuration:root-instance-title
   key ipv6_random slap-configuration:ipv6-random
   key ipv4_random slap-configuration:ipv4-random
 default-parameters =

software/theia/software.cfg

@@ -135,7 +135,7 @@ initialization =
     standalone.request(
         args.sr,
         "Embedded Instance",
-        args.srtype or None,
+        args.srtype,
         partition_parameter_kw=params,
     )
 

stack/erp5/buildout.cfg

@@ -7,6 +7,7 @@ extends =
   buildout.hash.cfg
   ../../component/fonts/buildout.cfg
   ../../component/git/buildout.cfg
+  ../../component/ghostscript/buildout.cfg
   ../../component/graphviz/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/xz-utils/buildout.cfg
@@ -65,8 +66,6 @@ parts +=
   slapos-cookbook
   mroonga-mariadb
   tesseract
-  tesseract-eng-traineddata
-  tesseract-osd-traineddata
   zabbix-agent
 
 # Buildoutish
@@ -252,6 +251,7 @@ link-binary =
   ${graphviz:location}/bin/dot
   ${grep:location}/bin/grep
   ${imagemagick:location}/bin/convert
+  ${ghostscript:location}/bin/gs
   ${imagemagick:location}/bin/identify
   ${jpegoptim:location}/bin/jpegoptim
   ${jsl:location}/bin/jsl

stack/slapos.cfg

@@ -190,7 +190,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.184
-slapos.core = 1.6.12
+slapos.core = 1.6.13
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20