default, download-unpacked: improve multiarch support

README.rst

@@ -24,16 +24,18 @@ Example that installs software::
     statlib:lib/libfoo.a
     statlib:lib/libfoo.la
     dynlib:bin/foo linked:libbar.so.1,libc.so.6,libfoo.so.1 rpath:${bar:location}/lib,!/lib
-  x86 = http://host/path/x86.zip [md5sum]
-  x86-64 =  http://host/path/x64.zip [md5sum]
+  i386-linux-gnu = http://host/path/i386.tar.gz [md5sum]
+  x86_64-linux-gnu = http://host/path/amd64.tar.gz [md5sum]
   install =
-    url, md5sum = options[guessPlatform()].split()
+    url, md5sum = options[multiarch()].split()
     extract_dir = self.extract(self.download(url, md5sum))
     self.copyTree(guessworkdir(extract_dir), location)
     ${:update}
   update =
     ...
 
+See also the ``download-unpacked`` recipe, which has options to install multiarch specific archives.
+
 Using the init option::
 
   [section-one]
@@ -507,6 +509,23 @@ Example::
   Uninstalling download.
   Installing download.
 
+multiarch
+---------
+
+`url` (optionally combined with `md5sum` and `alternate-url`) is actually a
+default option to specify what to download. It can be overridden with machine
+specific value, like in the example of the default recipe::
+
+  [buildout]
+  parts =
+    download
+
+  [download]
+  recipe = slapos.recipe.build:download-unpacked
+  i386-linux-gnu = http://host/path/i386.tar.gz [alternate-url] md5sum]
+  x86_64-linux-gnu = http://host/path/amd64.tar.gz [[alternate-url] md5sum]
+
+Values can be separated with newlines.
 
 ==============================
  slapos.recipe.build:gitclone

slapos/recipe/__init__.py

@@ -5,7 +5,7 @@ except ImportError:
     from pkgutil import extend_path
     __path__ = extend_path(__path__, __name__)
 
-import errno, json, logging, os, shutil, stat
+import errno, json, logging, os, shutil, stat, subprocess
 from hashlib import md5
 from zc.buildout import UserError
 from zc.buildout.rmtree import rmtree as buildout_rmtree
@@ -31,6 +31,22 @@ def make_read_only_recursively(path):
     for file_ in file_list:
       make_read_only(os.path.join(root, file_))
 
+def multiarch():
+  try:
+    m = multiarch.cache
+  except AttributeError:
+    try:
+      m = subprocess.check_output(
+        ('gcc', '-dumpmachine'),  universal_newlines=True).rstrip()
+    except Exception as e:
+      m = e
+    multiarch.cache = m
+  if type(m) is str:
+    if m:
+      return m
+    raise Exception("empty multiarch value")
+  raise m
+
 def rmtree(path):
   try:
     buildout_rmtree(path)

slapos/recipe/build/__init__.py

@@ -28,7 +28,6 @@ import errno
 import linecache
 import logging
 import os
-from platform import uname
 import shlex
 import shutil
 import subprocess
@@ -36,16 +35,9 @@ import sys
 import tempfile
 from setuptools import archive_util
 import zc.buildout
-from .. import is_true, rmtree, EnvironMixin, Shared
+from .. import is_true, multiarch, rmtree, EnvironMixin, Shared
 from ..downloadunpacked import extraction_drivers, patched_extraction_drivers
 
-ARCH_MAP = {
-    'i386': 'x86',
-    'i586': 'x86',
-    'i686': 'x86',
-    'x86_64': 'x86-64'
-}
-
 def readElfAsDict(f):
   """Reads ELF information from file"""
   result = subprocess.check_output(('readelf', '-d', f),
@@ -87,11 +79,8 @@ def guessworkdir(path):
   x = os.listdir(path)
   return os.path.join(path, *x) if len(x) == 1 else path
 
-def guessPlatform():
-  return ARCH_MAP[uname()[-2]]
-
 GLOBALS = (lambda *x: {x.__name__: x for x in x})(
-  call, guessPlatform, guessworkdir, is_true)
+  call, guessworkdir, is_true, multiarch)
 
 class Script(EnvironMixin):
   """Free script building system"""

slapos/recipe/downloadunpacked.py

@@ -31,7 +31,8 @@ import tarfile
 import tempfile
 from hashlib import md5
 from setuptools import archive_util
-from . import is_true, EnvironMixin, Shared
+from zc.buildout import UserError
+from . import is_true, multiarch, EnvironMixin, Shared
 from . import make_read_only_recursively # for slapos.recipe.cmmi
 
 class Recipe(EnvironMixin):
@@ -41,8 +42,25 @@ class Recipe(EnvironMixin):
     self.name = name
     self.options = options
     self._strip = is_true(options.get('strip-top-level-dir'), None)
-    self._url = options['url']
-
+    self._alternate = options.get('alternate-url')
+    self._md5sum = options.get('md5sum')
+    url = options.get('url')
+    try:
+      m = multiarch()
+    except Exception as e:
+      if not url:
+        raise UserError(
+          "option 'url' missing; could not fallback on multiarch (%s)" % e)
+    else:
+      x = options.get(m)
+      if x:
+        x = x.split()
+        url = x.pop(0)
+        self._md5sum = x.pop() if x else None
+        self._alternate, = x or (None,)
+      elif not url:
+        raise UserError("missing option: 'url' or %r" % m)
+    self._url = url
     shared = Shared(buildout, name, options)
     destination = options.get('destination')
     if destination:
@@ -63,9 +81,9 @@ class Recipe(EnvironMixin):
     #       inside slapos.buildout (see networkcache support)
     from zc.buildout import download
     location = self._shared.location
-    alternate = self.options.get('alternate-url')
+    alternate = self._alternate
     path, is_temp = download.Download(self.buildout['buildout'],
-      hash_name=True)(self._url, self.options.get('md5sum') or None,
+      hash_name=True)(self._url, self._md5sum or None,
                       **({'alternate_url': alternate} if alternate else {}))
     try:
       archive_util.extraction_drivers = patched_extraction_drivers

[Julien Muchembled]

May not be the best idea. This commit does improve things but I realize it may be even better to patch buildout so that the multiarch is available in the expression of conditional sections. For example, '[packer]' in slapos.git/component/packer/buildout.cfg would become:

[packer]
recipe = slapos.recipe.build:download-unpacked
version = 0.7.5
url = https://releases.hashicorp.com/packer/${:version}/packer_${:version}${:_url}.zip

[packer:multiarch()=='i386-linux-gnu']
_url = linux_386
md5sum = a545108a0ccfde7c1e74de6c4e6fdded

[packer:multiarch()=='x86_64-linux-gnu']
_url = linux_amd64
md5sum = f343d709b84db494e8d6ec38259aa4a6

[Julien Muchembled]

/cc @jerome @kazuhiko

[Jérome Perrin]

I agree, being able to use buildout's conditional section would be the best. platform module is available in namespace for conditions, but I see here we are using output of gcc -dumpmachine so I guess information provided by platform module is not always enough.