Commit a50c4470 by Łukasz Nowak

playbook: Setup ERP5 with frontend

By using separated frontend/backend requests and firewall configuration
allow to access ERP5 on public IPv4 of host on https port 443 and http
port 80.
1 parent 44c00301
......@@ -3,10 +3,17 @@
connection: local
vars:
software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.20/software/erp5/software.cfg
software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.48/software/erp5/software.cfg
frontend_software_release_url: https://lab.node.vifib.com/nexedi/slapos/raw/1.0.48/software/apache-frontend/software.cfg
startup_playbook_id: erp5-standalone.yml
playbook_name: erp5
request_instance_template: request-erp5-cluster.j2
request_instance_template: request-erp5.j2
request_frontend_template: request-erp5-frontend.j2
firewall_setup_template: setup-firewall.j2
frontend_master_reference: master-frn-{{ playbook_name }}
frontend_slave_reference: slave-srn-{{ playbook_name }}
backend_url_path: /tmp/playbook-{{ playbook_name }}-backend-url
public_ipv4_path: /tmp/playbook-{{ playbook_name }}-public_ipv4
roles:
- standalone-shared
......@@ -15,6 +15,10 @@
shell: "slapos configure local"
when: slapos_cfg.stat.exists == False
- name: Disable multimaster
shell: sed -i '/# Here goes the list of/,/\/some\/arbitrary\/local\/unix\/path/d' /etc/opt/slapos/slapos-proxy.cfg
ignore_errors: True
- name: Wait for proxy
wait_for: host=127.0.0.1 port=8080 delay=10
......
......@@ -11,6 +11,7 @@ import ssl
import urllib2
from xml.dom import minidom
import json
import hashlib
def fmt_date():
return time.strftime("%Y%m%d")
......@@ -20,39 +21,57 @@ def fmt_date():
def discover_software():
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT DISTINCT software_release FROM partition11")
qry = cur.execute("SELECT DISTINCT url FROM software11")
return [row[0] for row in qry if row[0]]
def get_connection_information(software_release):
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=?", (software_release,) )
if 'software/erp5' in software_release:
partition_reference = 'instance-of-erp5'
elif 'software/apache-frontend' in software_release:
partition_reference = 'apache-frontend-1'
else:
raise ValueError
qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=? AND partition_reference=?", (software_release, partition_reference) )
xml = None
for row in qry:
xml = str(row[0])
break
break
if xml is None:
print software_release
return (None, None)
instance = minidom.parseString(xml)
try:
el = instance.getElementsByTagName('parameter')[0]
value = el.childNodes[0].nodeValue
except:
return "error"
if not value.startswith("{"):
value = "\"" + value + "\""
json_text = json.loads(value)
if 'family-admin' in json_text:
return (json_text['family-admin'], json_text['inituser-password'])
elif 'insecure' in json_text:
return (json_text, None)
if partition_reference == 'instance-of-erp5':
try:
el = instance.getElementsByTagName('parameter')[0]
value = el.childNodes[0].nodeValue
except:
return "error"
if not value.startswith("{"):
value = "\"" + value + "\""
json_text = json.loads(value)
if 'family-admin' in json_text:
return (json_text['family-admin'], json_text['inituser-password'])
elif 'insecure' in json_text:
return (json_text, None)
else:
return (None, None)
elif partition_reference == 'apache-frontend-1':
ip = None
for el in instance.getElementsByTagName('parameter'):
if el.getAttribute('id') == 'private-ipv4':
if len(el.childNodes) > 0:
ip = str(el.childNodes[0].nodeValue) or None
break
return (ip, None)
else:
return (None, None)
raise ValueError
def check_tables():
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
......@@ -63,16 +82,22 @@ def check_tables():
print "tables aren't ready yet, your build may have failed, check logs in /opt/slapos/log/"
sys.exit(0)
def get_build_status():
def get_build_status(software_release):
sr_hash = hashlib.md5(software_release).hexdigest()
try:
f = open("/opt/slapos/log/slapos-node-software-" + fmt_date() + ".log")
except:
f = open("/opt/slapos/log/slapos-node-software.log")
lines = f.readlines()
if "Finished software releases" not in lines[-1]:
os.stat(os.path.join('/opt/slapgrid', sr_hash, '.completed'))
except OSError:
try:
f = open("/opt/slapos/log/slapos-node-software-" + fmt_date() + ".log")
except Exception:
try:
f = open("/opt/slapos/log/slapos-node-software.log")
except Exception:
return False
lines = f.readlines()
if "ERROR" in lines[-3]:
return "error"
return False
if "ERROR" in lines[-3]:
return "error"
return True
# Check if the last two lines show the software finished building.
......@@ -80,28 +105,56 @@ def get_build_status():
# Otherwise it passed and we can move on.
# We want to open today's log, as it is most up to date
def status(software_release):
build = get_build_status()
def status():
software_release_list = discover_software()
software_release_list_status = [get_build_status(software_release)
for software_release in software_release_list]
if len(software_release_list) == 2 and \
software_release_list_status == [True, True]:
build = True
else:
build = False
if build:
zope_ip, pw = get_connection_information(software_release)
erp5_sr = [q for q in software_release_list if 'software/erp5' in q][0]
frn_sr = [q for q in software_release_list
if 'software/apache-frontend' in q][0]
zope_ip, pw = get_connection_information(erp5_sr)
try:
ip = open('/tmp/playbook-erp5-public-ipv4').read()
except Exception:
frontend = None
else:
if len(ip) == 0:
frontend = zope_ip
else:
frontend = 'https://' + ip
frontend_ip, _ = get_connection_information(frn_sr)
if zope_ip is None or frontend is None or frontend_ip is None:
print "Build successful, please wait for instantiation"
sys.exit(2)
print ("Build successful, connect to:\n"
" " + zope_ip)
" " + frontend)
if pw is not None:
print (" with\n"
" username: zope password: " + pw)
elif not build:
print "Your software is still building, be patient it can take awhile"
sys.exit(2)
elif build == "error":
print "An error occurred while building, check /opt/slapos/log/slapos-node-software-" + \
fmt_date() + ".log for details"
sys.exit(2)
if 'error' in [software_release_list_status]:
print "An error occurred while building, check /opt/slapos/log/slapos-node-software-" + \
fmt_date() + ".log for details"
sys.exit(2)
else:
print "Your software is still building, be patient it can take a while"
sys.exit(2)
ipv6 = None
# check if the services are actually running (run slapos node and parse output)
if pw is None:
zope_ip = "https://" + zope_ip[zope_ip.index("@")+1:]
original_zope_ip = zope_ip
if "[" in zope_ip and "]" in zope_ip:
ipv6 = zope_ip[zope_ip.index("[")+1:zope_ip.index("]")]
......@@ -122,7 +175,7 @@ def status(software_release):
print ""
print "DEBUG information: %s" % e
sys.exit(2)
if r1.getcode() != 200:
print "At least one of your services isn't running! Check with slapos node"
print "restart a service with slapos node restart slappart:service"
......@@ -133,18 +186,18 @@ def status(software_release):
print "The URL above may require extra configuration if you want to access it"
print "from another machine. You can install an apache locally and include the"
print "the follow rewrite rule (http version):"
print """
print """
RewriteRule ^/(.*) %s/VirtualHostBase/http/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
or (https version):
RewriteRule ^/(.*) %s/VirtualHostBase/https/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
""" % (original_zope_ip, original_zope_ip)
""" % (original_zope_ip, original_zope_ip)
def info(software_release):
if get_build_status():
if get_build_status(software_release):
print get_connection_information(software_release)
else:
print "Information unavailable at this time, run " + sys.argv[0] + " -s for details"
......@@ -153,9 +206,9 @@ def usage():
print ("Get the status and information of your ERP5 build\n"
"Usage:")
print (" --help (-h): Print this message and exit\n"
" --status (-s): Print the status of the build\n"
" --info (-i): Print the partition tables\n"
" --dump (-d): Dump the entire database (alias for slapos proxy show)\n")
" --status (-s): Print the status of your ERP5 build\n"
" --info (-i): Print the technical information about partition tables\n"
" --dump (-d): Dump the entire database (alias for slapos proxy show)\n")
def dump():
subprocess.call(["slapos", "proxy", "show", "-u", "/opt/slapos/slapproxy.db"])
......@@ -177,14 +230,13 @@ def main(argv):
sys.exit()
elif opt in ("-s", "--status"):
check_tables()
for sr in discover_software():
status(sr)
status()
elif opt in ("-i", "--info"):
check_tables()
for sr in discover_software():
info(sr)
elif opt in ("-d", "--dump"):
dump()
if __name__ == "__main__":
main(sys.argv[1:])
---
- name: create partition script
- name: Fetch public IPv4
shell: ifconfig `route -n | grep '^0\.0\.0\.0' | head -n 1 | awk '{print $NF}'` | { IFS=' :';read r;read r r a r;echo $a; }
register: public_ipv4
- name: Store public IPv4 for other scripts
shell: echo "{{ public_ipv4.stdout }}" > /tmp/playbook-{{ playbook_name }}-public-ipv4
- name: create instance request script
template: src={{ request_instance_template }} dest=/tmp/playbook-request-{{ playbook_name }} mode=700
- name: Supply and Request ERP5 Cluster
shell: cat /tmp/playbook-request-{{ playbook_name }} | slapos console
cron:
cron_file=ansible-{{ playbook_name }}-request
user=root
name="ERP5 Instance Request"
minute=*/5
job="/bin/echo 'execfile(\"/tmp/playbook-request-{{ playbook_name }}\")' | /usr/bin/slapos console > /dev/null 2>&1"
- name: create frontend request script
template: src={{ request_frontend_template }} dest=/tmp/playbook-request-frontend-{{ playbook_name }} mode=700
- name: Supply and Request ERP5 Frontend
cron:
cron_file=ansible-{{ playbook_name }}-request-frontend
user=root
name="ERP5 Frontend Request"
minute=*/5
job="/bin/echo 'execfile(\"/tmp/playbook-request-frontend-{{ playbook_name }}\")' | /usr/bin/slapos console > /dev/null 2>&1"
- name: Create firewall script
template: src={{ firewall_setup_template }} dest=/tmp/playbook-firewall-setup-{{ playbook_name }} mode=700
- name: Setup firewall
cron:
cron_file=ansible-{{ playbook_name }}-firewall
user=root
name="Setup firewall"
minute=*/5
job="/tmp/playbook-firewall-setup-{{ playbook_name }} > /dev/null 2>&1"
- name: create erp5-show
copy: src=erp5-show dest=/usr/local/bin/erp5-show mode=755
......@@ -12,16 +46,27 @@
template: src=start-script.j2 dest=/usr/local/bin/{{ playbook_name }}-startup mode=755
- name: Include on reboot
cron: name="Start Up script" special_time=reboot job="/usr/local/bin/{{ playbook_name }}-startup >> /var/log/{{ playbook_name }}-startup.log 2>&1"
- name: Get slapos.playbook directory name
shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
register: tmp_dir
cron:
cron_file=ansible-{{ playbook_name }}-reboot
user=root
name="Start Up script"
special_time=reboot
job="/usr/local/bin/{{ playbook_name }}-startup >> /var/log/{{ playbook_name }}-startup.log 2>&1"
- name: Check if /opt/slapos.playbook already exists
stat: path=/opt/slapos.playbook/
register: playbook_state
- name: Remove /opt/slapos.playbook
file:
path: /opt/slapos.playbook
state: absent
when: playbook_dir != '/opt/slapos.playbook'
- name: Copy slapos.playbook
copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
when: playbook_state.stat.exists == False
copy: src={{ playbook_dir }}/ dest=/opt/slapos.playbook/
when: playbook_dir != '/opt/slapos.playbook'
- name: Check ERP5 state
shell: /usr/local/bin/erp5-show -s
register: erp5_state
- name: Expose ERP5
debug:
msg="{{ erp5_state.stdout_lines }}"
computer_id = 'local_computer'
frontend_software_release_url = '{{ frontend_software_release_url }}'
supply(frontend_software_release_url, computer_id)
public_ipv4 = '{{ public_ipv4.stdout }}'
# frontend master partition
request(software_release=frontend_software_release_url, partition_reference='{{ frontend_master_reference }}')
# frontend slave partition
backend_url = None
try:
backend_url = open('{{ backend_url_path }}', 'r').read()
except Exception:
pass
if backend_url is not None:
request(
software_release=frontend_software_release_url,
partition_reference='{{ frontend_slave_reference }}',
shared=True,
partition_parameter_kw={
'url': backend_url,
'type': 'zope',
'custom_domain': public_ipv4,
'server-alias': '*'
}
)
......@@ -38,11 +38,17 @@ parameter_dict = {
# Choose a title
title = "instance-of-{{ playbook_name }}"
request(software_url,
erp5_instance = request(software_url,
title,
filter_kw={'computer_guid': computer_id},
software_type='create-erp5-site',
partition_parameter_kw={
'_': json.dumps(parameter_dict, sort_keys=True, indent=2),
}
)
).getConnectionParameterDict()
if '_' in erp5_instance:
backend_dict = json.loads(erp5_instance['_'])
if 'family-admin-v6' in backend_dict:
backend_url = str(backend_dict['family-admin-v6'])
open('{{ backend_url_path }}', 'w').write(backend_url)
#!/bin/bash
export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
frontend_ipv4=`sqlite3 /opt/slapos/slapproxy.db "select address from partition_network11 where partition_reference=(select reference from partition11 where requested_by = (select reference from partition11 where partition_reference='{{ frontend_master_reference }}')) and netmask='255.255.255.255';"`
local_access_comment="Local {{ playbook_name }} IPv4 access"
remote_access_comment="Remote {{ playbook_name }} IPv4 access"
if [ ! -z "$frontend_ipv4" ] ; then
local_count=`iptables -t nat -vnL | egrep "${local_access_comment}.*to:$frontend_ipv4" | wc -l`
remote_count=`iptables -t nat -vnL | egrep "${remote_access_comment}.*to:$frontend_ipv4" | wc -l`
if [ $local_count == "2" ] && [ $remote_count == "2" ] ; then
exit 0
fi
iptables -t nat -vnL PREROUTING | grep dpt:443 | grep -q "${remote_access_comment}" && iptables -t nat -D PREROUTING `iptables -t nat -vnL PREROUTING --line-numbers | grep dpt:443 | grep "${remote_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL OUTPUT | grep dpt:443 | grep -q "${local_access_comment}" && iptables -t nat -D OUTPUT `iptables -t nat -vnL OUTPUT --line-numbers | grep dpt:443 | grep "${local_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL PREROUTING | grep dpt:80 | grep -q "${remote_access_comment}" && iptables -t nat -D PREROUTING `iptables -t nat -vnL PREROUTING --line-numbers | grep dpt:80 | grep "${remote_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -vnL OUTPUT | grep dpt:80 | grep -q "${local_access_comment}" && iptables -t nat -D OUTPUT `iptables -t nat -vnL OUTPUT --line-numbers | grep dpt:80 | grep "${local_access_comment}" | cut -d ' ' -f 1`
iptables -t nat -A OUTPUT -p tcp -d {{ public_ipv4.stdout }} --dport 443 -j DNAT --to $frontend_ipv4:4443 -m comment --comment "${local_access_comment}"
iptables -t nat -A PREROUTING -p tcp -d {{ public_ipv4.stdout }} --dport 443 -j DNAT --to-destination $frontend_ipv4:4443 -m comment --comment "${remote_access_comment}"
iptables -t nat -A OUTPUT -p tcp -d {{ public_ipv4.stdout }} --dport 80 -j DNAT --to $frontend_ipv4:8080 -m comment --comment "${local_access_comment}"
iptables -t nat -A PREROUTING -p tcp -d {{ public_ipv4.stdout }} --dport 80 -j DNAT --to-destination $frontend_ipv4:8080 -m comment --comment "${remote_access_comment}"
fi
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!