Commit 80eb130f authored by Kristopher Ruzic's avatar Kristopher Ruzic

Adds new recipe for packer, fixes issue with build and sets kvm to use generated image

a lot of the work done here is just duplicated from KVM's recipe. This is bad but works for now
parent d6aefc18
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
parts += parts +=
packer
dash dash
dash-output dash-output
slapos-cookbook
[packer] [packer]
recipe = slapos.recipe.build recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
x86-64).
url_x86-64 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_amd64.zip url_x86-64 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_amd64.zip
url_x86 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_386.zip url_x86 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_386.zip
......
...@@ -160,6 +160,7 @@ setup(name=name, ...@@ -160,6 +160,7 @@ setup(name=name,
'notifier.notify = slapos.recipe.notifier:Notify', 'notifier.notify = slapos.recipe.notifier:Notify',
'novnc = slapos.recipe.novnc:Recipe', 'novnc = slapos.recipe.novnc:Recipe',
'onetimeupload = slapos.recipe.onetimeupload:Recipe', 'onetimeupload = slapos.recipe.onetimeupload:Recipe',
'packer = slapos.recipe.packer:Recipe',
'pbs = slapos.recipe.pbs:Recipe', 'pbs = slapos.recipe.pbs:Recipe',
'postgres = slapos.recipe.postgres:Recipe', 'postgres = slapos.recipe.postgres:Recipe',
'postgres.export = slapos.recipe.postgres.backup:ExportRecipe', 'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',
......
This is in large part a duplication of the kvm recipe. Which is not a good thing, as that
recipe is not needed as it is. Working this together with whatever the future of the KVM recipe is
should be done.
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import sys
class Recipe(GenericBaseRecipe):
"""
packer instance configuration.
"""
def install(self):
# Sanitize drive type parameter
self.options.setdefault('disk-type', 'virtio')
if not self.options.get('disk-type') in ['ide', 'scsi', 'sd',
'mtd', 'floppy', 'pflash', 'virtio']:
print 'Warning: "disk-type" parameter is not in allowed values. Using ' \
'"virtio" value.'
self.options['disk-type'] = 'virtio'
self.options['python-path'] = sys.executable
path_list = []
if self.isTrueValue(self.options.get('use-nat')):
# XXX This could be done using Jinja.
for port in self.options['nat-rules'].split():
tunnel_port = int(port) + 10000
tunnel_path = self.createExecutable(
'%s-%s' % (self.options['6tunnel-wrapper-path'], tunnel_port),
self.substituteTemplate(
self.getTemplateFilename('6to4.in'),
{
'ipv6': self.options['ipv6'],
'ipv6_port': tunnel_port,
'ipv4': self.options['ipv4'],
'ipv4_port': tunnel_port,
'shell_path': self.options['shell-path'],
'6tunnel_path': self.options['6tunnel-path'],
},
),
)
path_list.append(tunnel_path)
runner_path = self.createExecutable(
self.options['runner-path'],
self.substituteTemplate(self.getTemplateFilename('packer_run.in'),
self.options))
path_list.append(runner_path)
controller_path = self.createExecutable(
self.options['controller-path'],
self.substituteTemplate(self.getTemplateFilename('packer_controller_run.in'),
self.options))
return path_list
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s
#!%(python-path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
# Echo client program
import socket
import time
# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '%(socket-path)s'
vnc_password = '%(vnc-passwd)s'
# Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False
while not connected:
try:
so.connect(socket_path)
except socket.error:
time.sleep(1)
else:
connected = True
data = so.recv(1024)
# Enable qmp
so.send('{ "execute": "qmp_capabilities" }')
data = so.recv(1024)
# Set VNC password
so.send('{ "execute": "change", ' \
'"arguments": { "device": "vnc", "target": "password", ' \
' "arg": "' + vnc_password + '" } }')
data = so.recv(1024)
# Finish
so.close()
#!%(python-path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import ast
import hashlib
import os
import socket
import subprocess
import urllib
import gzip
import shutil
from random import shuffle
import glob
import re
# dict of values packer expects in json
build = {}
# XXX: give all of this through parameter, don't use this as template, but as module
# use_tap and use_nat set as Bool as per http://stackoverflow.com/a/922374
build["disk_size"] = '%(disk-size)s'
build["disk_type"] = '%(disk-type)s'
build["hostname"] = '%(packer-hostname)'
build["domain"] = '%(packer-domain)'
socket_path = '%(socket-path)s'
nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
disk_path = '%(disk-path)s'
nat_rules = '%(nat-rules)s'.strip()
use_tap = literal_eval('%(use-tap)s')
use_nat = literal_eval('%(use-nat)s')
tap_interface = '%(tap-interface)s'
listen_ip = '%(ipv4)s'
mac_address = '%(mac-address)s'
tap_mac_address = '%(tap-mac-address)s'
smp_count = '%(smp-count)s'
smp_options = '%(smp-options)s'.strip()
numa_list = '%(numa)s'.split()
ram_size = '%(ram-size)s'
pid_file_path = '%(pid-file-path)s'
etc_directory = '%(etc-directory)s'.strip()
httpd_port = %(httpd-port)s
netcat_bin = '%(netcat-binary)s'.strip()
cluster_doc_host = '%(cluster-doc-host)s'
cluster_doc_port = %(cluster-doc-port)s
# iso url and md5 are hardcoded for now
build["iso_url"] =
"http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso",
build["iso_checksum"] = "1a311f9afb68d6365211b13b4342c40b"
build["iso_checksum_type"] = "md5"
# set basic params
build["name"] = '%(packer-name)s'
build["accelerator"] = "kvm" # explicitly set
build["http_directory"] = "http"
# ssh params (will create account on install)
build["ssh_username"] = '%(packer-username)'
build["ssh_password"] = '%(packer-password)'
# boot parameters
build["boot_wait"] = "2s" # warm up
# runs through install
build["boot_command"] = [
"<esc><wait><wait>",
"install auto ",
"debian-installer=en_US locale=en_US keymap=us ",
"netcfg/get_hostname={{hostname}} ",
"netcfg/get_domain={{domain}} ",
"fb=false debconf/frontend=noninteractive ",
"passwd/user-fullname={{user}} ",
"passwd/user-password={{password}} ",
"passwd/user-password-again={{password}} ",
"passwd/username={{user}} ",
"<enter>"
]
def getSocketStatus(host, port):
s = None
for res in socket.getaddrinfo(host, port,
socket.AF_UNSPEC, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
s = socket.socket(af, socktype, proto)
except socket.error, msg:
s = None
continue
try:
s.connect(sa)
except socket.error, msg:
s.close()
s = None
continue
break
return s
def gen_packer_json(build):
packer_json = {"builders": [build]}
json_file = open("packer.json", 'w')
json_file.write(packer_json)
json_file.close()
# Generate network parameters
tap_network_parameter = []
nat_network_parameter = []
numa_parameter = []
number = -1
if use_nat:
number += 1
rules = 'user,id=lan%%s,' %% number + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip,
int(port) + 10000, port) for port in nat_rules.split())
if httpd_port > 0:
rules += ',guestfwd=tcp:10.0.2.100:80-cmd:%%s %%s %%s' %% (netcat_bin,
listen_ip, httpd_port)
if cluster_doc_host and cluster_doc_port > 0:
rules += ',guestfwd=tcp:10.0.2.101:443-cmd:%%s %%s %%s' %% (netcat_bin,
cluster_doc_host, cluster_doc_port)
nat_network_parameter = ['-netdev', rules,
'-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
if use_tap:
number += 1
tap_network_parameter = ['-netdev',
'tap,id=lan%%s,ifname=%%s,script=no,downscript=no' %% (number,
tap_interface),
'-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, tap_mac_address)]
smp = smp_count
if smp_options:
for option in smp_options.split(','):
key, val = option.split('=')
if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
smp += ',%%s=%%s' %% (key, val)
kvm_argument_list += [
['-enable-kvm'], ['-smp', smp], ['-m', ram_size],
['-drive', 'file=%%s,if=%%s' %% (disk_path, build.get(disk_type))],
['-vnc', '%%s:1,ipv4,password' %% listen_ip],
['-boot', 'order=cd,menu=on'],
['-qmp', 'unix:%%s,server' %% socket_path],
['-vga', 'std'], ['-cpu', 'host'],
['-pidfile', pid_file_path],
]
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list:
if rgx.match(numa):
numa_parameter.append(['-numa', numa])
kvm_argument_list.append(numa_parameter)
if tap_network_parameter == [] and nat_network_parameter == []:
print 'Warning : No network interface defined.'
else:
kvm_argument_list.append(nat_network_parameter)
kvm_argument_list.append(tap_network_parameter)
build["qemuargs"] = kvm_argument_list
print 'Generating Packer json with qemu args: \n %%s' %% ' '.join(kvm_argument_list)
gen_packer_json(build)
{ {
"variables": {
"user": "nexedi",
"password": "test",
"disk_size": {{ disk_size }},
"domain": ""
},
"builders": "builders":
[ [
{ {
...@@ -14,17 +7,16 @@ ...@@ -14,17 +7,16 @@
"type": "qemu", "type": "qemu",
"format": "qcow2", "format": "qcow2",
"accelerator": "kvm", "accelerator": "kvm",
"disk_size": "{{ user `disk_size`}}", "disk_size": "{{disk_size}}",
"iso_url": "iso_url": "http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso",
"http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso", "iso_checksum": "1a311f9afb68d6365211b13b4342c40b",
"iso_checksum": "0b31bccccb048d20b551f70830bb7ad0",
"iso_checksum_type": "md5", "iso_checksum_type": "md5",
"http_directory": "http", "http_directory": "http",
"ssh_username": "{{user `user`}}", "ssh_username": "{{user}}",
"ssh_password": "{{user `password`}}", "ssh_password": "{{password}}",
"shutdown_command": "echo '{{user `password`}}'|sudo -S shutdown -h now", "shutdown_command": "echo '{{password}}'|sudo -S shutdown -h now",
"boot_wait": "2s", "boot_wait": "2s",
"boot_command": [ "boot_command": [
...@@ -32,15 +24,15 @@ ...@@ -32,15 +24,15 @@
"install auto ", "install auto ",
"preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ",
"debian-installer=en_US locale=en_US keymap=us ", "debian-installer=en_US locale=en_US keymap=us ",
"netcfg/get_hostname={{ .Name }} ", "netcfg/get_hostname={{hostname}} ",
"netcfg/get_domain={{ user `domain`}} ", "netcfg/get_domain={{domain}} ",
"fb=false debconf/frontend=noninteractive ", "fb=false debconf/frontend=noninteractive ",
"passwd/user-fullname={{user `user`}} ", "passwd/user-fullname={{user}} ",
"passwd/user-password={{user `password`}} ", "passwd/user-password={{password}} ",
"passwd/user-password-again={{user `password`}} ", "passwd/user-password-again={{password}} ",
"passwd/username={{user `user`}} ", "passwd/username={{user}} ",
"<enter>" "<enter>"
] ]
......
[buildout] [buildout]
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
extends =
${template:output}
parts = parts =
packer-json-template directory
packer-instance
packer-build-template packer-build-template
dynamic-template-kvm
[slap-parameter]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/ etc = $${buildout:directory}/etc
etc-run = $${:etc}/run bin = ${buildout:directory}/bin
srv = $${buildout:directory}/srv/ srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
log = $${:var}/log
scripts = $${:etc}/run
services = $${:etc}/service
promises = $${:etc}/promise
novnc-conf = $${:etc}/novnc
run = $${:var}/run
ca-dir = $${:srv}/ssl
public = $${:srv}/public/
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
[create-mac]
recipe = slapos.cookbook:generate.mac
storage-path = $${directory:srv}/mac
[create-tap-mac]
recipe = slapos.cookbook:generate.mac
storage-path = $${directory:srv}/tap_mac
[gen-passwd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:srv}/passwd
bytes = 8
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -28,35 +58,66 @@ context = ...@@ -28,35 +58,66 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
$${:extra-context} $${:extra-context}
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
configuration.packer_json = ${:_profile_base_location_}/debian8.json.jinja2
[packer-json-template]
recipe = slapos.recipe.template
url = $${instance-parameter:configuration.packer_json}
output = ${buildout:parts-directory}/packer-template.json
context =
key disk_size slap-parameter:disk-size
#destination = ${buildout:parts-directory}
#md5sum = 47d492dafe5cb314bdc49bf013d21ead
[packer-build-template] [packer-build-template]
< = jinja2-template-base < = jinja2-template-base
template = ${packer-template-wrapper:output} template = ${packer-template-wrapper:output}
rendered = $${directory:etc-run}/packer-build rendered = $${directory:scripts}/packer-build
mode = 0700 mode = 0700
extra-context = extra-context =
key env packer-configuration:packer-environment key env packer-configuration:packer-environment
key content packer-configuration:packer-build-command key content packer-configuration:packer-build-command
[packer-configuration] [packer-configuration]
disk-size = 10000
user = nexedi
password = test
hostname =
domain =
packer-environment = ${buildout:parts-directory}/qemu/bin/ packer-environment = ${buildout:parts-directory}/qemu/bin/
packer-build-command = packer-build-command =
${buildout:parts-directory}/packer/packer build -debug -color=false $${packer-json-template:output} > ${buildout:parts-directory}/packer/packer build -debug -color=false ${buildout:parts-directory}/packer.json > testing.log
testing.log
[packer-instance]
recipe = slapos.cookbook:packer
vnc-passwd = $${gen-passwd:passwd}
ipv4 = $${slap-parameter:ipv4-random}
ipv6 = $${slap-parameter:ipv6-random}
vnc-ip = $${:ipv4}
vnc-port = 5901
default-disk-image =
nbd-host = ${slap-parameter:nbd-host}
nbd-port = ${slap-parameter:nbd-port}
nbd2-host = ${slap-parameter:nbd2-host}
nbd2-port = ${slap-parameter:nbd2-port}
tap-interface =
disk-size = ""
disk-type = ${slap-parameter:disk-type}
socket-path = $${directory:var}/qmp_socket
pid-file-path = $${directory:run}/pid_file
smp-count = ${slap-parameter:cpu-count}
smp-options = ${slap-parameter:cpu-options}
ram-size = ${slap-parameter:ram-size}
numa = ${slap-parameter:numa}
mac-address = $${create-mac:mac-address}
tap-mac-address = $${create-tap-mac:mac-address}
# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
runner-path = $${directory:services}/packer
controller-path = $${directory:scripts}/packer_controller
use-tap = ${slap-parameter:use-tap}
use-nat = ${slap-parameter:use-nat}
nat-rules = ${slap-parameter:nat-rules}
6tunnel-wrapper-path = $${directory:services}/6tunnel
shell-path = {{ dash_executable_location }}
6tunnel-path = {{ sixtunnel_executable_location }}
...@@ -4,13 +4,15 @@ extends = ...@@ -4,13 +4,15 @@ extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../../component/qemu-kvm/buildout.cfg ../../component/qemu-kvm/buildout.cfg
../../component/packer/buildout.cfg ../../component/packer/buildout.cfg
../../software/kvm/software.cfg ../../component/6tunnel/buildout.cfg
../../component/lxml-python/buildout.cfg
parts += parts +=
template slapos-cookbook
template-kvm
qemu
instance-profile instance-profile
lxml-python
packer
qemu
[instance-profile] [instance-profile]
...@@ -19,8 +21,50 @@ url = ${:_profile_base_location_}/instance.cfg.in ...@@ -19,8 +21,50 @@ url = ${:_profile_base_location_}/instance.cfg.in
#md5sum = cf67212d3155767d0d0d8a6d75d2d8ad #md5sum = cf67212d3155767d0d0d8a6d75d2d8ad
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
[packer-template-wrapper] [packer-template-wrapper]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg output = ${buildout:directory}/template-wrapper.cfg
mode = 0644 mode = 0644
context =
raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
raw dash_executable_location ${dash-output:dash}
[slap-parameter]
# Default values if not specified
frontend-software-type = frontend
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
frontend-instance-guid =
frontend-instance-name = VNC Frontend
nbd-port = 1024
nbd-host =
nbd2-port = 1024
nbd2-host =
ram-size = 1024
disk-size = 10
disk-type = virtio
cpu-count = 1
# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
cpu-options =
# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
numa =
nat-rules = 22 80 443
use-nat = True
use-tap = False
virtual-hard-drive-url =
virtual-hard-drive-md5sum =
virtual-hard-drive-gzipped = False
external-disk-number = 0
external-disk-size = 20
external-disk-format = qcow2
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment