Adds new recipe for packer, fixes issue with build and sets kvm to use generated image

a lot of the work done here is just duplicated from KVM's recipe. This is bad but works for now

component/packer/buildout.cfg

 [buildout]
 
 extends =
-  ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
   
 parts +=
-  packer
   dash
   dash-output
-  slapos-cookbook
 
   
 [packer]
 recipe = slapos.recipe.build
 
-# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. 
-x86-64).
+# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
 url_x86-64 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_amd64.zip
 url_x86 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_386.zip
   

setup.py

@@ -160,6 +160,7 @@ setup(name=name,
           'notifier.notify = slapos.recipe.notifier:Notify',
           'novnc = slapos.recipe.novnc:Recipe',
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
+          'packer = slapos.recipe.packer:Recipe', 
           'pbs = slapos.recipe.pbs:Recipe',
           'postgres = slapos.recipe.postgres:Recipe',
           'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',

slapos/recipe/packer/README

+This is in large part a duplication of the kvm recipe. Which is not a good thing, as that 
+recipe is not needed as it is. Working this together with whatever the future of the KVM recipe is
+should be done.
\ No newline at end of file

slapos/recipe/packer/__init__.py

+##############################################################################
+#
+# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import os
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  packer instance configuration.
+  """
+  def install(self):
+    # Sanitize drive type parameter
+    self.options.setdefault('disk-type', 'virtio')
+    if not self.options.get('disk-type') in ['ide', 'scsi', 'sd',
+        'mtd', 'floppy', 'pflash', 'virtio']:
+      print 'Warning: "disk-type" parameter is not in allowed values. Using ' \
+          '"virtio" value.'
+      self.options['disk-type'] = 'virtio'
+
+    self.options['python-path'] = sys.executable
+
+    path_list = []
+
+    if self.isTrueValue(self.options.get('use-nat')):
+      # XXX This could be done using Jinja.
+      for port in self.options['nat-rules'].split():
+        tunnel_port = int(port) + 10000
+        tunnel_path = self.createExecutable(
+            '%s-%s' % (self.options['6tunnel-wrapper-path'], tunnel_port),
+            self.substituteTemplate(
+                self.getTemplateFilename('6to4.in'),
+                {
+                    'ipv6': self.options['ipv6'],
+                    'ipv6_port': tunnel_port,
+                    'ipv4': self.options['ipv4'],
+                    'ipv4_port': tunnel_port,
+                    'shell_path': self.options['shell-path'],
+                    '6tunnel_path': self.options['6tunnel-path'],
+                },
+            ),
+        )
+        path_list.append(tunnel_path)
+
+    runner_path = self.createExecutable(
+        self.options['runner-path'],
+        self.substituteTemplate(self.getTemplateFilename('packer_run.in'),
+                                self.options))
+    path_list.append(runner_path)
+
+    controller_path = self.createExecutable(
+      self.options['controller-path'],
+      self.substituteTemplate(self.getTemplateFilename('packer_controller_run.in'),
+                              self.options))
+    return path_list

slapos/recipe/packer/template/6to4.in

+#!%(shell_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s
+

slapos/recipe/packer/template/packer_controller_run.in

+#!%(python-path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+# Echo client program
+import socket
+import time
+
+# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
+
+socket_path = '%(socket-path)s'
+vnc_password = '%(vnc-passwd)s'
+
+# Connect to KVM qmp socket
+so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+connected = False
+while not connected:
+  try:
+    so.connect(socket_path)
+  except socket.error:
+    time.sleep(1)
+  else:
+    connected = True
+data = so.recv(1024)
+
+# Enable qmp
+so.send('{ "execute": "qmp_capabilities" }')
+data = so.recv(1024)
+
+# Set VNC password
+so.send('{ "execute": "change", ' \
+        '"arguments": { "device": "vnc", "target": "password", ' \
+        '               "arg": "' + vnc_password + '" } }')
+data = so.recv(1024)
+
+# Finish
+so.close()

slapos/recipe/packer/template/packer_run.in

+#!%(python-path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import ast
+import hashlib
+import os
+import socket
+import subprocess
+import urllib
+import gzip
+import shutil
+from random import shuffle
+import glob
+import re
+
+# dict of values packer expects in json
+build = {}
+
+# XXX: give all of this through parameter, don't use this as template, but as module
+# use_tap and use_nat set as Bool as per http://stackoverflow.com/a/922374
+build["disk_size"] = '%(disk-size)s'
+build["disk_type"] = '%(disk-type)s'
+build["hostname"] = '%(packer-hostname)'
+build["domain"] = '%(packer-domain)'
+
+socket_path = '%(socket-path)s'
+nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
+disk_path = '%(disk-path)s'
+nat_rules = '%(nat-rules)s'.strip()
+use_tap = literal_eval('%(use-tap)s')
+use_nat = literal_eval('%(use-nat)s')
+tap_interface = '%(tap-interface)s'
+listen_ip = '%(ipv4)s'
+mac_address = '%(mac-address)s'
+tap_mac_address = '%(tap-mac-address)s'
+smp_count = '%(smp-count)s'
+smp_options = '%(smp-options)s'.strip()
+numa_list = '%(numa)s'.split()
+ram_size = '%(ram-size)s'
+pid_file_path = '%(pid-file-path)s'
+etc_directory = '%(etc-directory)s'.strip()
+httpd_port = %(httpd-port)s
+netcat_bin = '%(netcat-binary)s'.strip()
+cluster_doc_host = '%(cluster-doc-host)s'
+cluster_doc_port = %(cluster-doc-port)s
+
+# iso url and md5 are hardcoded for now
+build["iso_url"] = 
+  "http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso",
+build["iso_checksum"] = "1a311f9afb68d6365211b13b4342c40b"
+build["iso_checksum_type"] = "md5"
+
+# set basic params
+build["name"] = '%(packer-name)s'
+build["accelerator"] = "kvm" # explicitly set
+build["http_directory"] = "http"
+
+# ssh params (will create account on install)
+build["ssh_username"] = '%(packer-username)'
+build["ssh_password"] = '%(packer-password)'
+
+# boot parameters
+build["boot_wait"] = "2s" # warm up
+
+# runs through install
+build["boot_command"] = [
+  "<esc><wait><wait>",
+  "install auto ",
+  "debian-installer=en_US locale=en_US keymap=us ",
+  "netcfg/get_hostname={{hostname}} ",
+  "netcfg/get_domain={{domain}} ",
+  
+  "fb=false debconf/frontend=noninteractive ",
+  
+  "passwd/user-fullname={{user}} ",
+  "passwd/user-password={{password}} ",
+  "passwd/user-password-again={{password}} ",
+  "passwd/username={{user}} ",
+  
+  "<enter>"
+]
+
+
+def getSocketStatus(host, port):
+  s = None
+  for res in socket.getaddrinfo(host, port,
+      socket.AF_UNSPEC, socket.SOCK_STREAM):
+    af, socktype, proto, canonname, sa = res
+    try:
+      s = socket.socket(af, socktype, proto)
+    except socket.error, msg:
+      s = None
+      continue
+    try:
+      s.connect(sa)
+    except socket.error, msg:
+      s.close()
+      s = None
+      continue
+    break
+  return s
+
+def gen_packer_json(build):
+  packer_json = {"builders": [build]}
+
+  json_file = open("packer.json", 'w')
+  json_file.write(packer_json)
+  json_file.close()
+
+
+# Generate network parameters
+tap_network_parameter = []
+nat_network_parameter = []
+numa_parameter = []
+number = -1
+if use_nat:
+  number += 1
+  rules = 'user,id=lan%%s,' %% number + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip,
+                        int(port) + 10000, port) for port in nat_rules.split())
+  if httpd_port > 0:
+    rules += ',guestfwd=tcp:10.0.2.100:80-cmd:%%s %%s %%s' %% (netcat_bin,
+                                                        listen_ip, httpd_port)
+  if cluster_doc_host and cluster_doc_port > 0:
+    rules += ',guestfwd=tcp:10.0.2.101:443-cmd:%%s %%s %%s' %% (netcat_bin,
+                                           cluster_doc_host, cluster_doc_port)
+  nat_network_parameter = ['-netdev', rules,
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
+if use_tap:
+  number += 1
+  tap_network_parameter = ['-netdev',
+          'tap,id=lan%%s,ifname=%%s,script=no,downscript=no' %% (number,
+            tap_interface),
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, tap_mac_address)]
+
+smp = smp_count
+if smp_options:
+  for option in smp_options.split(','):
+    key, val = option.split('=')
+    if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
+      smp += ',%%s=%%s' %% (key, val)
+
+kvm_argument_list += [
+  ['-enable-kvm'], ['-smp', smp], ['-m', ram_size], 
+  ['-drive', 'file=%%s,if=%%s' %% (disk_path, build.get(disk_type))],
+  ['-vnc', '%%s:1,ipv4,password' %% listen_ip],
+  ['-boot', 'order=cd,menu=on'],
+  ['-qmp', 'unix:%%s,server' %% socket_path],
+  ['-vga', 'std'], ['-cpu', 'host'],
+  ['-pidfile', pid_file_path],
+]
+
+rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
+for numa in numa_list:
+  if rgx.match(numa):
+    numa_parameter.append(['-numa', numa])
+kvm_argument_list.append(numa_parameter)
+
+if tap_network_parameter == [] and nat_network_parameter == []:
+  print 'Warning : No network interface defined.'
+else:
+  kvm_argument_list.append(nat_network_parameter)
+  kvm_argument_list.append(tap_network_parameter)
+
+
+build["qemuargs"] = kvm_argument_list
+
+print 'Generating Packer json with qemu args: \n %%s' %% ' '.join(kvm_argument_list)
+gen_packer_json(build)
+
+

software/packer/debian8.json.jinja2

 {
-    "variables": {
-        "user": "nexedi",
-        "password": "test",
-        "disk_size": {{ disk_size }},
-        "domain": ""
-    },
-
     "builders":
     [
         {
@@ -14,17 +7,16 @@
             "type": "qemu",
             "format": "qcow2",
             "accelerator": "kvm",
-            "disk_size": "{{ user `disk_size`}}",
+            "disk_size": "{{disk_size}}",
 
-            "iso_url": 
-"http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso",
-            "iso_checksum": "0b31bccccb048d20b551f70830bb7ad0",
+            "iso_url": "http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso",
+            "iso_checksum": "1a311f9afb68d6365211b13b4342c40b",
             "iso_checksum_type": "md5",
 
             "http_directory": "http",
-            "ssh_username": "{{user `user`}}",
-            "ssh_password": "{{user `password`}}",
-            "shutdown_command": "echo '{{user `password`}}'|sudo -S shutdown -h now",
+            "ssh_username": "{{user}}",
+            "ssh_password": "{{password}}",
+            "shutdown_command": "echo '{{password}}'|sudo -S shutdown -h now",
 
             "boot_wait": "2s",
             "boot_command": [
@@ -32,15 +24,15 @@
                    "install auto ",
                    "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ",
                    "debian-installer=en_US locale=en_US keymap=us ",
-                   "netcfg/get_hostname={{ .Name }} ",
-                   "netcfg/get_domain={{ user `domain`}} ",
+                   "netcfg/get_hostname={{hostname}} ",
+                   "netcfg/get_domain={{domain}} ",
 
                    "fb=false debconf/frontend=noninteractive ",
 
-                   "passwd/user-fullname={{user `user`}} ",
-                   "passwd/user-password={{user `password`}} ",
-                   "passwd/user-password-again={{user `password`}} ",
-                   "passwd/username={{user `user`}} ",
+                   "passwd/user-fullname={{user}} ",
+                   "passwd/user-password={{password}} ",
+                   "passwd/user-password-again={{password}} ",
+                   "passwd/username={{user}} ",
 
                    "<enter>"
             ]

software/packer/instance.cfg.in

 [buildout]
 
-
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
-extends = 
-  ${template:output}
-
 parts =
-  packer-json-template
+  directory
+  packer-instance
   packer-build-template
-  dynamic-template-kvm
+
+[slap-parameter]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-etc-run = $${:etc}/run
-srv = $${buildout:directory}/srv/
+etc = $${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+novnc-conf = $${:etc}/novnc
+run = $${:var}/run
+ca-dir = $${:srv}/ssl
+public = $${:srv}/public/
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+
+[create-mac]
+recipe = slapos.cookbook:generate.mac
+storage-path = $${directory:srv}/mac
+
+[create-tap-mac]
+recipe = slapos.cookbook:generate.mac
+storage-path = $${directory:srv}/tap_mac
+
+[gen-passwd]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:srv}/passwd
+bytes = 8
+
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -28,35 +58,66 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     $${:extra-context}
 
-[instance-parameter]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
-configuration.packer_json = ${:_profile_base_location_}/debian8.json.jinja2
-
-[packer-json-template]
-recipe = slapos.recipe.template
-url = $${instance-parameter:configuration.packer_json}
-output = ${buildout:parts-directory}/packer-template.json
-context =
-  key disk_size slap-parameter:disk-size
-#destination = ${buildout:parts-directory}
-#md5sum = 47d492dafe5cb314bdc49bf013d21ead
-
 [packer-build-template]
 < = jinja2-template-base
 template = ${packer-template-wrapper:output}
-rendered = $${directory:etc-run}/packer-build
+rendered = $${directory:scripts}/packer-build
 mode = 0700
 extra-context =
   key env packer-configuration:packer-environment
   key content packer-configuration:packer-build-command
   
 [packer-configuration]
+disk-size = 10000
+user = nexedi
+password = test
+hostname = 
+domain = 
 packer-environment = ${buildout:parts-directory}/qemu/bin/ 
 packer-build-command = 
-  ${buildout:parts-directory}/packer/packer build -debug -color=false $${packer-json-template:output}  > 
-testing.log
+  ${buildout:parts-directory}/packer/packer build -debug -color=false ${buildout:parts-directory}/packer.json  > testing.log
+
+
+[packer-instance]
+recipe = slapos.cookbook:packer
+
+vnc-passwd = $${gen-passwd:passwd}
+
+ipv4 = $${slap-parameter:ipv4-random}
+ipv6 = $${slap-parameter:ipv6-random}
+vnc-ip = $${:ipv4}
+
+vnc-port = 5901
+
+default-disk-image = 
+nbd-host = ${slap-parameter:nbd-host}
+nbd-port = ${slap-parameter:nbd-port}
+nbd2-host = ${slap-parameter:nbd2-host}
+nbd2-port = ${slap-parameter:nbd2-port}
+
+tap-interface = 
+
+disk-size = ""
+disk-type = ${slap-parameter:disk-type}
+
+socket-path = $${directory:var}/qmp_socket
+pid-file-path = $${directory:run}/pid_file
+
+smp-count = ${slap-parameter:cpu-count}
+smp-options = ${slap-parameter:cpu-options}
+ram-size = ${slap-parameter:ram-size}
+numa = ${slap-parameter:numa}
+mac-address = $${create-mac:mac-address}
+tap-mac-address = $${create-tap-mac:mac-address}
+
+# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
+runner-path = $${directory:services}/packer
+controller-path = $${directory:scripts}/packer_controller
+
+use-tap = ${slap-parameter:use-tap}
+use-nat = ${slap-parameter:use-nat}
+nat-rules = ${slap-parameter:nat-rules}
+6tunnel-wrapper-path = $${directory:services}/6tunnel
+
+shell-path = {{ dash_executable_location }}
+6tunnel-path = {{ sixtunnel_executable_location }}

software/packer/software.cfg

@@ -4,13 +4,15 @@ extends =
   ../../stack/slapos.cfg
   ../../component/qemu-kvm/buildout.cfg
   ../../component/packer/buildout.cfg
-  ../../software/kvm/software.cfg
-    
+  ../../component/6tunnel/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+
 parts +=
-  template
-  template-kvm
-  qemu
+  slapos-cookbook
   instance-profile
+  lxml-python
+  packer
+  qemu
 
 
 [instance-profile]
@@ -19,8 +21,50 @@ url = ${:_profile_base_location_}/instance.cfg.in
 #md5sum = cf67212d3155767d0d0d8a6d75d2d8ad
 output = ${buildout:directory}/instance.cfg
 
+
 [packer-template-wrapper]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/wrapper.in
 output = ${buildout:directory}/template-wrapper.cfg
 mode = 0644
+context =
+  raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+  raw dash_executable_location ${dash-output:dash} 
+
+[slap-parameter]
+# Default values if not specified
+frontend-software-type = frontend
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
+frontend-instance-guid =
+frontend-instance-name = VNC Frontend
+nbd-port = 1024
+nbd-host =
+nbd2-port = 1024
+nbd2-host =
+
+ram-size = 1024
+disk-size = 10
+disk-type = virtio
+
+cpu-count = 1
+# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
+cpu-options = 
+# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
+numa = 
+
+nat-rules = 22 80 443
+use-nat = True
+use-tap = False
+
+virtual-hard-drive-url =
+virtual-hard-drive-md5sum =
+virtual-hard-drive-gzipped = False
+
+external-disk-number = 0
+external-disk-size = 20
+external-disk-format = qcow2
+
+
+[versions]
+plone.recipe.command = 1.1
+slapos.recipe.template = 2.8