Cloudooo: Libreoffice 7.5.2.2 / Python 3

see nexedi/cloudooo!32

See merge request nexedi/slapos!1394

component/avahi/buildout.cfg

+# Avahi - Service Discovery for Linux using mDNS/DNS-SD -- compatible with Bonjour
+[buildout]
+extends =
+  ../dbus/buildout.cfg
+  ../glib/buildout.cfg
+  ../pkgconfig/buildout.cfg
+
+parts =
+  avahi
+
+[avahi]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/lathiat/avahi/releases/download/v0.8/avahi-0.8.tar.gz
+md5sum = 229c6aa30674fc43c202b22c5f8c2be7
+configure-options =
+  --disable-static
+  --disable-libevent
+  --disable-mono
+  --disable-monodoc
+  --disable-python
+  --disable-qt3
+  --disable-qt4
+  --disable-qt5
+  --disable-gtk
+  --disable-gtk3
+  --disable-libdaemon
+  --disable-core-docs
+  --with-distro=none
+  --with-systemdsystemunitdir=no
+environment =
+  PATH=${pkgconfig:location}/bin:${glib:location}/bin:%(PATH)s
+  CFLAGS=-I${gdbm:location}/include
+  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig

component/cloudooo/buildout.cfg

 [buildout]
 extends =
+  ../git/buildout.cfg
   ../lxml-python/buildout.cfg
 
 parts =
@@ -12,9 +13,8 @@ setup = ${cloudooo-repository:location}
 [cloudooo-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/cloudooo.git
-branch = master
 git-executable = ${git:location}/bin/git
-revision = 78d15e091df750952998a6a23f49300a5a0faa53
+revision = 98055878283583fad61374c93fb777c7a91fa070
 
 [cloudooo]
 recipe = zc.recipe.egg

component/file/buildout.cfg

@@ -11,8 +11,8 @@ extends =
 [file]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.icm.edu.pl/packages/file/file-5.41.tar.gz
-md5sum = 18233bb0a0089dfdc7dfbc93b96f231b
+url = http://ftp.astron.com/pub/file/file-5.44.tar.gz
+md5sum = a60d586d49d015d842b9294864a89c7a
 configure-options =
   --disable-static
   --disable-libseccomp

component/libreoffice-bin/buildout.cfg

@@ -8,11 +8,11 @@ parts =
 
 [libreoffice-bin]
 recipe = slapos.recipe.build
-version = 5.2.4.2
-url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
+version = 7.5.2.2
+url = https://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
 
 # where office code can be found?
-officedir = libreoffice5.2
+officedir = libreoffice7.5
 install =
   import os
   import sys
@@ -38,12 +38,14 @@ install =
 cpio = ${cpio:location}/bin/cpio
 rpm2cpio = ${rpm2cpio:target}
 
-[libreoffice-bin:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
-_url1 = x86
-_url2 = Linux_x86
-md5sum = 7a0b33a2d18f06143258c428c32de213
+[libreoffice-bin:getattr(sys,'_multiarch',None)!='x86_64-linux-gnu']
+recipe = plone.recipe.command
+command =
+  echo unsupported architecture
+  false
+stop-on-error = true
 
 [libreoffice-bin:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
 _url1 = x86_64
 _url2 = Linux_x86-64
-md5sum = cbea6cd17063b5bcbe0cb32f7819f0cf
+md5sum = efb0351ac9ebd954c43f77d59632bd28

component/rpm2cpio/buildout.cfg

@@ -6,5 +6,5 @@ parts =
 # https://github.com/ruda/rpm2cpio
 recipe = slapos.recipe.build:download
 shared = true
-url = ${:_profile_base_location_}/${:_buildout_section_name_}
-md5sum = aa3a5920a1d8963592be0c2666ee05e2
+url = https://raw.githubusercontent.com/ruda/rpm2cpio/5afad4b65e4661a771db6f728abefe2c1e84b9ae/rpm2cpio.py
+md5sum = 1cb52f9030304c2b7625657b2ef2379e

component/rpm2cpio/rpm2cpio

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Standalone RPM to CPIO converter
-# Copyright (c) 2012 Rudá Moura
-# https://github.com/ruda/rpm2cpio
-#
-# Impove gzip header detection thanks to
-# http://afb.users.sourceforge.net/centos/rpm2cpio.py
-#
-# Copyright (C) 1997,1998,1999, Roger Espel Llima
-# Copyright (C) 2000, Sergey Babkin
-# Copyright (C) 2009, Alex Kozlov
-# Copyright (C) 2010, Anders F Bjorklund
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and any associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# SOFTWARE'S COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE
-
-'''Extract cpio archive from RPM package.
-
-rpm2cpio converts the RPM on standard input or first parameter to a CPIO archive on standard output.
-
-Usage:
-rpm2cpio < adjtimex-1.20-2.1.i386.rpm  | cpio -it
-./sbin/adjtimex
-./usr/share/doc/adjtimex-1.20
-./usr/share/doc/adjtimex-1.20/COPYING
-./usr/share/doc/adjtimex-1.20/COPYRIGHT
-./usr/share/doc/adjtimex-1.20/README
-./usr/share/man/man8/adjtimex.8.gz
-133 blocks
-'''
-
-import sys
-import struct
-import StringIO
-import gzip
-
-RPM_MAGIC = '\xed\xab\xee\xdb'
-GZIP_MAGIC = '\x1f\x8b'
-
-def rpm2cpio(stream_in=sys.stdin, stream_out=sys.stdout):
-    lead = stream_in.read(96)
-    if lead[0:4] != RPM_MAGIC:
-        raise IOError, 'the input is not a RPM package'
-    lead = stream_in.read(16)
-    if not lead:
-        raise IOError, 'No header'
-
-    while True:
-        (magic, ignore, sections, bytes) = struct.unpack("!LLLL", lead)
-        (smagic, smagic2) = struct.unpack("!HL", lead[0:6])
-
-        if smagic == 0x1f8b:
-            break
-
-        # skip the headers
-        stream_in.seek(16 * sections + bytes, 1)
-        while True:
-            lead = stream_in.read(1)
-            if lead == "":
-                raise IOError, 'No header'
-            if (0,) == struct.unpack("B", lead):
-                continue
-            break
-        lead += stream_in.read(15)
-        if lead == "":
-            raise IOError, 'No header'
-
-    stream_in.seek(-len(lead), 1)
-    gzipper = gzip.GzipFile(fileobj=stream_in)
-    data = gzipper.read()
-    stream_out.write(data)
-
-if __name__ == '__main__':
-    if sys.argv[1:]:
-        try:
-            fin = open(sys.argv[1])
-            rpm2cpio(fin)
-            fin.close()
-        except IOError, e:
-            print 'Error:', sys.argv[1], e
-    else:
-        try:
-            rpm2cpio()
-        except IOError, e:
-            print 'Error:', e

software/cloudooo/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = d1e4d7306c39f2ebc64d0407860d4301
 
 [template-cloudooo-instance]
 filename = instance-cloudooo.cfg.in
-md5sum = 3c499fd3cdfc7915d6eaf1cf4130b56d
+md5sum = 13759bf9720f0e7109fc35a8ad8a50a9
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

software/cloudooo/instance-cloudooo-input-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
-  "extends": "./schema-definitions.json#",
   "properties": {
     "tcpv4-port": {
-      "allOf": [
-        {
-          "$ref": "#/definitions/tcpv4port"
-        },
-        {
-          "description": "Start allocating ports at this value, going upward",
-          "default": 23000
-        }
-      ]
+      "description": "TCP port used for HTTP server",
+      "default": 8000,
+      "type": "integer"
     },
     "backend-count": {
       "description": "Number of backend cloudooo instances",
@@ -20,7 +13,7 @@
       "type": "integer"
     },
     "timeout": {
-      "description": "Configure apache with this timeout",
+      "description": "Timeout for document conversions, in seconds",
       "type": "integer"
     },
     "mimetype-entry-addition": {

software/cloudooo/instance-cloudooo.cfg.in

@@ -10,7 +10,7 @@
 {%   set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%}
 {% endif -%}
 {% do assert(next_port > 0) -%}
-{% set next_port = itertools.count(next_port).next -%}
+{% set next_port = itertools.count(next_port).__next__ -%}
 
 {% set backend_count = instance_parameter_dict['backend-count'] | int -%}
 {% if instance_parameter_dict.get('backend-count-parameter-name') -%}
@@ -85,7 +85,7 @@ output = ${directory:apache-conf}/apache.conf
 context = section parameter_dict apache-conf-parameter-dict
 
 [apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict.values()) }}
+backend-list = {{ dumps(list(apache_dict.values())) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
 error-log = ${directory:log}/apache-error.log
@@ -103,7 +103,7 @@ ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 <= monitor-promise-base
 promise = check_url_available
 name = apache.py
-config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
+config-url = https://{{ ipv4 }}:{{ list(apache_dict.values())[0][0] }}
 # XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
 # is that we don't have a "503 Service Unavailable" from apache or haproxy.
 config-http-code = 400
@@ -155,7 +155,7 @@ includes =
 recipe = slapos.cookbook:generic.cloudooo
 ip = {{ ipv4 }}
 environment =
-  LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
+  LD_LIBRARY_PATH = {{ parameter_dict['avahi'] }}/lib:{{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['nss'] }}/lib:{{ parameter_dict['nspr'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-conf:output}
   PATH = ${binary-link:target-directory}
   LANG = C.UTF-8

software/cloudooo/software-common.cfg

@@ -40,6 +40,7 @@ context =
 
 [dynamic-template-cloudooo-instance-parameter-dict]
 apache = ${apache:location}
+avahi = ${avahi:location}
 buildout-bin-directory = ${buildout:bin-directory}
 cairo = ${cairo:location}
 coreutils = ${coreutils:location}
@@ -80,6 +81,8 @@ libpng12 = ${libpng12:location}
 libreoffice-bin = ${libreoffice-bin:location}
 libxcb = ${libxcb:location}
 mesa = ${mesa:location}
+nss = ${nss:location}
+nspr = ${nspr:location}
 openssl = ${openssl:location}
 onlyoffice-core = ${onlyoffice-core:location}
 poppler = ${poppler:location}
@@ -104,4 +107,4 @@ url = ${:_profile_base_location_}/${:filename}
 
 [versions]
 argparse = 1.4.0
-pyPdf = 1.13
+pypdf = 3.6.0:whl

software/cloudooo/software.cfg

@@ -6,9 +6,6 @@ parts =
   ${cloudooo-buildout:parts}
   template-cloudooo
 
-[python]
-part = python2.7
-
 [cloudooo-software-parameter-dict]
 publish-url-name = cloudooo
 port-parameter-name = tcpv4-port

software/cloudooo/test/test.py

@@ -195,7 +195,7 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
 
 class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
   __partition_reference__ = 'lo'
-  pdf_producer = 'LibreOffice 5.2'
+  pdf_producer = 'LibreOffice 7.5'
   expected_font_mapping = {
       'Arial': 'LiberationSans',
       'Arial Black': 'DejaVuSans',
@@ -223,7 +223,7 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
       'Liberation Sans Narrow': 'LiberationSansNarrow',
       'Liberation Serif': 'LiberationSerif',
       'Linux LibertineG': 'LinuxLibertineG',
-      'OpenSymbol': 'OpenSymbol',
+      'OpenSymbol': {'OpenSymbol', 'IPAMincho'},
       'Palatino': 'DejaVuSerif',
       'Roboto Black': 'Roboto-Black',
       'Roboto Condensed Light': 'RobotoCondensed-Light',

stack/cloudooo.cfg

 [buildout]
 extends =
   ../stack/slapos.cfg
+  ../component/avahi/buildout.cfg
   ../component/apache/buildout.cfg
   ../component/cloudooo/buildout.cfg
   ../component/coreutils/buildout.cfg
@@ -22,12 +23,12 @@ extends =
   ../component/logrotate/buildout.cfg
   ../component/lxml-python/buildout.cfg
   ../component/mesa/buildout.cfg
+  ../component/nss/buildout.cfg
   ../component/onlyoffice-core/buildout.cfg
+  ../component/openssl/buildout.cfg
   ../component/poppler/buildout.cfg
   ../component/xorg/buildout.cfg
 
-versions = versions
-
 parts =
   ${stack-cloudooo-buildout:parts}
 
@@ -74,5 +75,5 @@ Paste = 3.4.0
 PasteScript = 3.2.0:whl
 WSGIUtils = 0.7.2
 WSGIserver = 1.3
-python-magic = 0.4.18
+python-magic = 0.4.27
 PasteDeploy = 2.1.0

[Kazuhiko Shiozaki]

@jerome why this change is needed ? well, I'm also wondering why we had 'OpenSymbol': 'OpenSymbol' before.

[Jérome Perrin]

What this test does is using this expected_font_mapping ({k: v}) to convert html -> pdf with an html text using the font k and inspects what are the fonts (v) used in the PDF.

For this specific case, libreoffice 5 converting this html to PDF:

      <style>
          p { font-family: "OpenSymbol"; font-size: 20pt; }
      </style>
      <p>the quick brown fox jumps over the lazy dog.</p>
      <p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>

created a PDF using only OpenSymbol, with libreoffice 7, the PDF uses OpenSymbol and IPAMincho.

Frankly I did not check much, but my guess is this might just be a bug fixed in libreoffice, Opensymbol does not have glyphs for alphabet letters, as we can see in https://fontsgeek.com/fonts/OpenSymbol-Regular so maybe now libreoffice automatically includes a font which can render these characters.

The purpose of this test was to check that we use fonts correctly (not something wrong like ignoring the fontconfig file and using liberation for everything), so when I thought that there is only a diff for symbol fonts I was assuming it's OK.