...
 
Commits (4)
......@@ -83,6 +83,16 @@ In order to test one image (after unzip), you can run:
bash util/quick-test packer-centos72 "-display sdl"
Watch out. quick-test will NOT write data into the image, so after exiting,
the image will be reverted back to its original state.
In order to have your work saved using quick test, please use information from
qemu man page:
-snapshot
Write to temporary files instead of disk image files. In this case, the raw disk image you use is not written back. You can however force the write back by
pressing C-a s.
Extra
-----
......
......@@ -16,8 +16,8 @@
"accelerator": "kvm",
"disk_size": "{{ user `disk_size`}}",
"iso_url": "http://cdimage.debian.org/debian-cd/8.3.0/amd64/iso-cd/debian-8.3.0-amd64-netinst.iso",
"iso_checksum": "a9b490b4215d1e72e876b031dafa7184",
"iso_url": "http://cdimage.debian.org/debian-cd/8.7.1/amd64/iso-cd/debian-8.7.1-amd64-netinst.iso",
"iso_checksum": "453312bf56fc45669fec5ebc0f025ac7",
"iso_checksum_type": "md5",
"http_directory": "http",
......
......@@ -16,8 +16,8 @@
"accelerator": "kvm",
"disk_size": "{{ user `disk_size`}}",
"iso_url": "http://cdimage.debian.org/debian-cd/8.6.0/amd64/iso-cd/debian-8.6.0-amd64-netinst.iso",
"iso_checksum": "e9f61bf327db6d8f7cee05a99f2353cc",
"iso_url": "http://cdimage.debian.org/debian-cd/8.7.1/amd64/iso-cd/debian-8.7.1-amd64-netinst.iso",
"iso_checksum": "453312bf56fc45669fec5ebc0f025ac7",
"iso_checksum_type": "md5",
"http_directory": "http",
......
......@@ -4,28 +4,28 @@ cat << EOF > /root/run-test
BOOTSTRAP="/root/start-bootstrap"
FILE="/etc/cron.d/ansible-vm-bootstrap"
if [[ ! -f "$BOOTSTRAP" ]]
if [[ ! -f "\$BOOTSTRAP" ]]
then
exit 1
fi
# Check if playbook has been correctly extracted
COUNT=$(ls /opt/slapos.playbook | wc -l)
if [[ ! $COUNT -gt 1 ]]
COUNT=\$(ls /opt/slapos.playbook | wc -l)
if [[ ! \$COUNT -gt 1 ]]
then
rm -f $FILE
rm -f \$FILE
rm -f /opt/slapos.playbook/playbook.tar.gz
fi
lf=/tmp/pidLockFile
cat /dev/null >> $lf
read lastPID < $lf
cat /dev/null >> \$lf
read lastPID < \$lf
# if lastPID is not null and a process with that pid exists , exit
[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
[ ! -z "\$lastPID" -a -d /proc/\$lastPID ] && exit
# save my pid in the lock file
echo $$ > $lf
echo \$\$ > \$lf
if [[ -f "$FILE" ]] && [[ -s "$FILE" ]] && [[ $COUNT -gt 1 ]]
if [[ -f "\$FILE" ]] && [[ -s "\$FILE" ]] && [[ \$COUNT -gt 1 ]]
then
rm /etc/cron.d/vm-boostrap
fi
......
#/bin/bash -e
# Quick command to test the VM image with output file.
/usr/bin/qemu-system-x86_64 $2 -drive file=$1,if=virtio,cache=writeback,discard=ignore -boot once=d -m 512M -name test -machine type=pc,accel=kvm -netdev user,id=user.0,hostfwd=tcp::3213-:22 -device virtio-net,netdev=user.0 -vnc 0.0.0.0:47
echo "Changes will not be written"
echo "To force write please use:"
echo "man qemu: You can however force the write back by pressing C-a s."
/usr/bin/qemu-system-x86_64 $2 -snapshot -drive file=$1,if=virtio,cache=writeback,discard=ignore -boot once=d -m 512M -name test -machine type=pc,accel=kvm -netdev user,id=user.0,hostfwd=tcp::3213-:22 -device virtio-net,netdev=user.0 -vnc 0.0.0.0:47
......@@ -3,10 +3,17 @@
connection: local
vars:
software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.20/software/erp5/software.cfg
software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.48/software/erp5/software.cfg
frontend_software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.48/software/apache-frontend/software.cfg
startup_playbook_id: erp5-standalone.yml
playbook_name: erp5
request_instance_template: request-erp5-cluster.j2
request_instance_template: request-erp5.j2
request_frontend_template: request-erp5-frontend.j2
firewall_setup_template: setup-firewall.j2
frontend_master_reference: master-frn-{{ playbook_name }}
frontend_slave_reference: slave-srn-{{ playbook_name }}
backend_url_path: /tmp/playbook-{{ playbook_name }}-backend-url
public_ipv4_path: /tmp/playbook-{{ playbook_name }}-public_ipv4
roles:
- standalone-shared
......@@ -15,6 +15,10 @@
shell: "slapos configure local"
when: slapos_cfg.stat.exists == False
- name: Disable multimaster
shell: sed -i '/# Here goes the list of/,/\/some\/arbitrary\/local\/unix\/path/d' /etc/opt/slapos/slapos-proxy.cfg
ignore_errors: True
- name: Wait for proxy
wait_for: host=127.0.0.1 port=8080 delay=10
......
......@@ -11,6 +11,7 @@ import ssl
import urllib2
from xml.dom import minidom
import json
import hashlib
def fmt_date():
return time.strftime("%Y%m%d")
......@@ -20,39 +21,57 @@ def fmt_date():
def discover_software():
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT DISTINCT software_release FROM partition11")
qry = cur.execute("SELECT DISTINCT url FROM software11")
return [row[0] for row in qry if row[0]]
def get_connection_information(software_release):
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=?", (software_release,) )
if 'software/erp5' in software_release:
partition_reference = 'instance-of-erp5'
elif 'software/apache-frontend' in software_release:
partition_reference = 'apache-frontend-1'
else:
raise ValueError
qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=? AND partition_reference=?", (software_release, partition_reference) )
xml = None
for row in qry:
xml = str(row[0])
break
break
if xml is None:
print software_release
return (None, None)
instance = minidom.parseString(xml)
try:
el = instance.getElementsByTagName('parameter')[0]
value = el.childNodes[0].nodeValue
except:
return "error"
if not value.startswith("{"):
value = "\"" + value + "\""
json_text = json.loads(value)
if 'family-admin' in json_text:
return (json_text['family-admin'], json_text['inituser-password'])
elif 'insecure' in json_text:
return (json_text, None)
if partition_reference == 'instance-of-erp5':
try:
el = instance.getElementsByTagName('parameter')[0]
value = el.childNodes[0].nodeValue
except:
return "error"
if not value.startswith("{"):
value = "\"" + value + "\""
json_text = json.loads(value)
if 'family-admin' in json_text:
return (json_text['family-admin'], json_text['inituser-password'])
elif 'insecure' in json_text:
return (json_text, None)
else:
return (None, None)
elif partition_reference == 'apache-frontend-1':
ip = None
for el in instance.getElementsByTagName('parameter'):
if el.getAttribute('id') == 'private-ipv4':
if len(el.childNodes) > 0:
ip = str(el.childNodes[0].nodeValue) or None
break
return (ip, None)
else:
return (None, None)
raise ValueError
def check_tables():
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
......@@ -63,16 +82,22 @@ def check_tables():
print "tables aren't ready yet, your build may have failed, check logs in /opt/slapos/log/"
sys.exit(0)
def get_build_status():
def get_build_status(software_release):
sr_hash = hashlib.md5(software_release).hexdigest()
try:
f = open("/opt/slapos/log/slapos-node-software-" + fmt_date() + ".log")
except:
f = open("/opt/slapos/log/slapos-node-software.log")
lines = f.readlines()
if "Finished software releases" not in lines[-1]:
os.stat(os.path.join('/opt/slapgrid', sr_hash, '.completed'))
except OSError:
try:
f = open("/opt/slapos/log/slapos-node-software-" + fmt_date() + ".log")
except Exception:
try:
f = open("/opt/slapos/log/slapos-node-software.log")
except Exception:
return False
lines = f.readlines()
if "ERROR" in lines[-3]:
return "error"
return False
if "ERROR" in lines[-3]:
return "error"
return True
# Check if the last two lines show the software finished building.
......@@ -80,28 +105,56 @@ def get_build_status():
# Otherwise it passed and we can move on.
# We want to open today's log, as it is most up to date
def status(software_release):
build = get_build_status()
def status():
software_release_list = discover_software()
software_release_list_status = [get_build_status(software_release)
for software_release in software_release_list]
if len(software_release_list) == 2 and \
software_release_list_status == [True, True]:
build = True
else:
build = False
if build:
zope_ip, pw = get_connection_information(software_release)
erp5_sr = [q for q in software_release_list if 'software/erp5' in q][0]
frn_sr = [q for q in software_release_list
if 'software/apache-frontend' in q][0]
zope_ip, pw = get_connection_information(erp5_sr)
try:
ip = open('/tmp/playbook-erp5-public-ipv4').read()
except Exception:
frontend = None
else:
if len(ip) == 0:
frontend = zope_ip
else:
frontend = 'https://' + ip
frontend_ip, _ = get_connection_information(frn_sr)
if zope_ip is None or frontend is None or frontend_ip is None:
print "Build successful, please wait for instantiation"
sys.exit(2)
print ("Build successful, connect to:\n"
" " + zope_ip)
" " + frontend)
if pw is not None:
print (" with\n"
" username: zope password: " + pw)
elif not build:
print "Your software is still building, be patient it can take awhile"
sys.exit(2)
elif build == "error":
print "An error occurred while building, check /opt/slapos/log/slapos-node-software-" + \
fmt_date() + ".log for details"
sys.exit(2)
if 'error' in [software_release_list_status]:
print "An error occurred while building, check /opt/slapos/log/slapos-node-software-" + \
fmt_date() + ".log for details"
sys.exit(2)
else:
print "Your software is still building, be patient it can take a while"
sys.exit(2)
ipv6 = None
# check if the services are actually running (run slapos node and parse output)
if pw is None:
zope_ip = "https://" + zope_ip[zope_ip.index("@")+1:]
original_zope_ip = zope_ip
if "[" in zope_ip and "]" in zope_ip:
ipv6 = zope_ip[zope_ip.index("[")+1:zope_ip.index("]")]
......@@ -122,7 +175,7 @@ def status(software_release):
print ""
print "DEBUG information: %s" % e
sys.exit(2)
if r1.getcode() != 200:
print "At least one of your services isn't running! Check with slapos node"
print "restart a service with slapos node restart slappart:service"
......@@ -133,18 +186,18 @@ def status(software_release):
print "The URL above may require extra configuration if you want to access it"
print "from another machine. You can install an apache locally and include the"
print "the follow rewrite rule (http version):"
print """
print """
RewriteRule ^/(.*) %s/VirtualHostBase/http/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
or (https version):
RewriteRule ^/(.*) %s/VirtualHostBase/https/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
""" % (original_zope_ip, original_zope_ip)
""" % (original_zope_ip, original_zope_ip)
def info(software_release):
if get_build_status():
if get_build_status(software_release):
print get_connection_information(software_release)
else:
print "Information unavailable at this time, run " + sys.argv[0] + " -s for details"
......@@ -153,9 +206,9 @@ def usage():
print ("Get the status and information of your ERP5 build\n"
"Usage:")
print (" --help (-h): Print this message and exit\n"
" --status (-s): Print the status of the build\n"
" --info (-i): Print the partition tables\n"
" --dump (-d): Dump the entire database (alias for slapos proxy show)\n")
" --status (-s): Print the status of your ERP5 build\n"
" --info (-i): Print the technical information about partition tables\n"
" --dump (-d): Dump the entire database (alias for slapos proxy show)\n")
def dump():
subprocess.call(["slapos", "proxy", "show", "-u", "/opt/slapos/slapproxy.db"])
......@@ -177,14 +230,13 @@ def main(argv):
sys.exit()
elif opt in ("-s", "--status"):
check_tables()
for sr in discover_software():
status(sr)
status()
elif opt in ("-i", "--info"):
check_tables()
for sr in discover_software():
info(sr)
elif opt in ("-d", "--dump"):
dump()
if __name__ == "__main__":
main(sys.argv[1:])
from setuptools import setup, find_packages
version = '0.0'
name = 'standalone-shared-tests'
setup(name=name,
version=version,
tests_require=['requests'],
test_suite='tests'
)
---
- name: create partition script
- name: Fetch public IPv4
shell: ifconfig `route -n | grep '^0\.0\.0\.0' | head -n 1 | awk '{print $NF}'` | { IFS=' :';read r;read r r a r;echo $a; }
register: public_ipv4
- name: Store public IPv4 for other scripts
shell: echo "{{ public_ipv4.stdout }}" > /tmp/playbook-{{ playbook_name }}-public-ipv4
- name: create instance request script
template: src={{ request_instance_template }} dest=/tmp/playbook-request-{{ playbook_name }} mode=700
- name: Supply and Request ERP5 Cluster
shell: cat /tmp/playbook-request-{{ playbook_name }} | slapos console
cron:
cron_file=ansible-{{ playbook_name }}-request
user=root
name="ERP5 Instance Request"
minute=*/5
job="/bin/echo 'execfile(\"/tmp/playbook-request-{{ playbook_name }}\")' | /usr/bin/slapos console > /dev/null 2>&1"
- name: create frontend request script
template: src={{ request_frontend_template }} dest=/tmp/playbook-request-frontend-{{ playbook_name }} mode=700
- name: Supply and Request ERP5 Frontend
cron:
cron_file=ansible-{{ playbook_name }}-request-frontend
user=root
name="ERP5 Frontend Request"
minute=*/5
job="/bin/echo 'execfile(\"/tmp/playbook-request-frontend-{{ playbook_name }}\")' | /usr/bin/slapos console > /dev/null 2>&1"
- name: Create firewall script
template: src={{ firewall_setup_template }} dest=/tmp/playbook-firewall-setup-{{ playbook_name }} mode=700
- name: Setup firewall
cron:
cron_file=ansible-{{ playbook_name }}-firewall
user=root
name="Setup firewall"
minute=*/5
job="/tmp/playbook-firewall-setup-{{ playbook_name }} > /dev/null 2>&1"
- name: create erp5-show
copy: src=erp5-show dest=/usr/local/bin/erp5-show mode=755
......@@ -12,16 +46,27 @@
template: src=start-script.j2 dest=/usr/local/bin/{{ playbook_name }}-startup mode=755
- name: Include on reboot
cron: name="Start Up script" special_time=reboot job="/usr/local/bin/{{ playbook_name }}-startup >> /var/log/{{ playbook_name }}-startup.log 2>&1"
- name: Get slapos.playbook directory name
shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
register: tmp_dir
cron:
cron_file=ansible-{{ playbook_name }}-reboot
user=root
name="Start Up script"
special_time=reboot
job="/usr/local/bin/{{ playbook_name }}-startup >> /var/log/{{ playbook_name }}-startup.log 2>&1"
- name: Check if /opt/slapos.playbook already exists
stat: path=/opt/slapos.playbook/
register: playbook_state
- name: Remove /opt/slapos.playbook
file:
path: /opt/slapos.playbook
state: absent
when: playbook_dir != '/opt/slapos.playbook'
- name: Copy slapos.playbook
copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
when: playbook_state.stat.exists == False
copy: src={{ playbook_dir }}/ dest=/opt/slapos.playbook/
when: playbook_dir != '/opt/slapos.playbook'
- name: Check ERP5 state
shell: /usr/local/bin/erp5-show -s
register: erp5_state
- name: Expose ERP5
debug:
msg="{{ erp5_state.stdout_lines }}"
computer_id = 'local_computer'
frontend_software_release_url = '{{ frontend_software_release_url }}'
supply(frontend_software_release_url, computer_id)
public_ipv4 = '{{ public_ipv4.stdout }}'
# frontend master partition
request(software_release=frontend_software_release_url, partition_reference='{{ frontend_master_reference }}')
# frontend slave partition
backend_url = None
try:
backend_url = open('{{ backend_url_path }}', 'r').read()
except Exception:
pass
if backend_url is not None:
request(
software_release=frontend_software_release_url,
partition_reference='{{ frontend_slave_reference }}',
shared=True,
partition_parameter_kw={
'url': backend_url,
'type': 'zope',
'custom_domain': public_ipv4,
'server-alias': '*'
}
)
......@@ -38,11 +38,17 @@ parameter_dict = {
# Choose a title
title = "instance-of-{{ playbook_name }}"
request(software_url,
erp5_instance = request(software_url,
title,
filter_kw={'computer_guid': computer_id},
software_type='create-erp5-site',
partition_parameter_kw={
'_': json.dumps(parameter_dict, sort_keys=True, indent=2),
}
)
).getConnectionParameterDict()
if '_' in erp5_instance:
backend_dict = json.loads(erp5_instance['_'])
if 'family-admin-v6' in backend_dict:
backend_url = str(backend_dict['family-admin-v6'])
open('{{ backend_url_path }}', 'w').write(backend_url)
#!/bin/bash
export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
frontend_ipv4=`sqlite3 /opt/slapos/slapproxy.db "select address from partition_network11 where partition_reference=(select reference from partition11 where requested_by = (select reference from partition11 where partition_reference='{{ frontend_master_reference }}')) and netmask='255.255.255.255';"`
local_access_comment="Local {{ playbook_name }} IPv4 access"
remote_access_comment="Remote {{ playbook_name }} IPv4 access"
if [ ! -z "$frontend_ipv4" ] ; then
local_count=`iptables -t nat -vnL | egrep "${local_access_comment}.*to:$frontend_ipv4" | wc -l`
remote_count=`iptables -t nat -vnL | egrep "${remote_access_comment}.*to:$frontend_ipv4" | wc -l`
if [ $local_count == "2" ] && [ $remote_count == "2" ] ; then
exit 0
fi
iptables -t nat -vnL PREROUTING | grep dpt:443 | grep -q "${remote_access_comment}" && iptables -t nat -D PREROUTING `iptables -t nat -vnL PREROUTING --line-numbers | grep dpt:443 | grep "${remote_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL OUTPUT | grep dpt:443 | grep -q "${local_access_comment}" && iptables -t nat -D OUTPUT `iptables -t nat -vnL OUTPUT --line-numbers | grep dpt:443 | grep "${local_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL PREROUTING | grep dpt:80 | grep -q "${remote_access_comment}" && iptables -t nat -D PREROUTING `iptables -t nat -vnL PREROUTING --line-numbers | grep dpt:80 | grep "${remote_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL OUTPUT | grep dpt:80 | grep -q "${local_access_comment}" && iptables -t nat -D OUTPUT `iptables -t nat -vnL OUTPUT --line-numbers | grep dpt:80 | grep "${local_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -A OUTPUT -p tcp -d {{ public_ipv4.stdout }} --dport 443 -j DNAT --to $frontend_ipv4:4443 -m comment --comment "${local_access_comment}"
iptables -t nat -A PREROUTING -p tcp -d {{ public_ipv4.stdout }} --dport 443 -j DNAT --to-destination $frontend_ipv4:4443 -m comment --comment "${remote_access_comment}"
iptables -t nat -A OUTPUT -p tcp -d {{ public_ipv4.stdout }} --dport 80 -j DNAT --to $frontend_ipv4:8080 -m comment --comment "${local_access_comment}"
iptables -t nat -A PREROUTING -p tcp -d {{ public_ipv4.stdout }} --dport 80 -j DNAT --to-destination $frontend_ipv4:8080 -m comment --comment "${remote_access_comment}"
fi
import requests
import unittest
import os
class TestSiteHttp(unittest.TestCase):
"""Check that configuration generated in the machine works"""
def setUp(self):
self.http_url = os.environ['TEST_ACCESS_URL_HTTP']
self.https_url = os.environ['TEST_ACCESS_URL_HTTPS']
def test_http_erp5_login_form(self):
"""Check that accessing site over HTTP redirects to HTTPS"""
result = requests.get(self.http_url + '/erp5/login_form', verify=False)
self.assertTrue(result.ok)
self.assertTrue(result.is_redirect)
self.assertTrue(result.is_permanent_redirect)
self.assertEqual(result.status_code, 302)
self.assertTrue(result.headers['Location'].endswith('/erp5/login_form'))
self.assertTrue(result.headers['Location'].startswith('https://'))
import requests
import unittest
import os
class TestSiteHttps(unittest.TestCase):
"""Check that configuration generated in the machine works"""
def setUp(self):
self.http_url = os.environ['TEST_ACCESS_URL_HTTP']
self.https_url = os.environ['TEST_ACCESS_URL_HTTPS']
def test_https_erp5_login_form(self):
"""Check that accessing login_form over HTTPS works"""
result = requests.get(self.https_url + '/erp5/login_form', verify=False)
self.assertTrue(result.ok)
self.assertFalse(result.is_redirect)
self.assertFalse(result.is_permanent_redirect)
self.assertEqual(result.status_code, 200)
self.assertTrue('ERP5 Free Open Source ERP and CRM' in result.text)
def test_https_erp5(self):
"""Check that accessing site over HTTPS redirects to login_form"""
result = requests.get(
self.https_url + '/erp5/', verify=False, allow_redirects=False)
self.assertTrue(result.ok)
self.assertTrue(result.is_redirect)
self.assertFalse(result.is_permanent_redirect)
self.assertEqual(result.status_code, 302)
self.assertTrue(result.headers['Location'].endswith('/erp5/login_form'))
import unittest
import os
import json
class TestSiteStatus(unittest.TestCase):
"""Asserts site status"""
def setUp(self):
self.site_status_json = os.environ['TEST_SITE_STATUS_JSON']
self.status_dict = json.load(open(self.site_status_json))
def test_build(self):
"""Checks that site was correctly created"""
# expose output for debugging
print 'Standard output:'
print self.status_dict['stdout']
print 'Standard error:'
print self.status_dict['stderr']
# Assert success
self.assertTrue(self.status_dict['success'])
def test_build_time_less_6h(self):
# Check that site was built in less than 6h
self.assertLess(self.status_dict['duration'], (3600. * 6))