Commit 23ff603b by Rafael Monnerat

Merge branch 'master' into 'master'

Improvements and fixes

See merge request !6
2 parents e47a0f5f c7f3d32f
......@@ -3,8 +3,10 @@
connection: local
vars:
software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1/software/erp5/software.cfg
software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1.1/software/erp5/software.cfg
startup_playbook_id: erp5-standalone.yml
playbook_name: erp5
request_instance_template: request-erp5-cluster.j2
roles:
- erp5-standalone
- standalone-shared
......@@ -23,6 +23,9 @@
- name: generate scripts
template: src=roles/install-script/templates/dev.j2 dest={{ base_path }}/install/devbook mode=0666
- name: generate ansible-only
template: src=roles/install-script/templates/ansible-only.j2 dest={{ base_path }}/install/ansible mode=0666
- name: generate index.html
copy: src=roles/install-script/files/index.html dest={{ base_path }}/install/index.html mode=0666
......@@ -47,3 +50,4 @@
- { role: "install-script", playbook_yml: "imt-vm-bootstrap.yml", script_path: "install/imt/vm-bootstrap"}
- { role: "install-script", playbook_yml: "imt-vm-cloudera-manager.yml", script_path: "install/imt/vm-cloudera-manager"}
- { role: "install-script", playbook_yml: "imt-re6stnet.yml", script_path: "install/imt/re6st"}
- { role: "install-script", playbook_yml: "test-suite.yml", script_path: "install/test-suite"}
......@@ -12,6 +12,6 @@
default: "noname"
roles:
- re6stnet-legacy
- { role: re6stnet-legacy, package_state: present }
- { role: package, package_name: ntp, package_state: present }
#!/bin/bash
# Reruns the ansible playbook, does nothing else
PLAYBOOK_ROOT=/opt/slapos.playbook/playbook/
PLAYBOOK_FILE=erp5-standalone.yml
cd $PLAYBOOK_ROOT # cd into the playbook directory
echo "Starting Ansible playbook:"
ansible-playbook $PLAYBOOK_FILE -i hosts --connection=local
---
- name: Supply erp5 software release
shell: slapos supply {{ software_release_url }} local_computer
- name: create partition script
copy: src=request-erp5-cluster dest=/tmp/playbook-request-erp5-cluster mode=700
- name: Request ERP5 Cluster
shell: cat /tmp/playbook-request-erp5-cluster | slapos console
- name: Add startup script
copy: src=erp5-startup dest=/usr/local/bin/erp5-startup mode=755
- name: Add to rc.local
lineinfile:
dest=/etc/rc.local insertbefore=BOF
line='bash /usr/local/bin/erp5-startup &'
state=present
- name: Get slapos.playbook directory name
shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
register: tmp_dir
- name: Check if /opt/slapos.playbook already exists
stat: path=/opt/slapos.playbook/
register: playbook_state
- name: Copy slapos.playbook
copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
when: playbook_state.stat.exists == False
......@@ -62,10 +62,10 @@ function trueorfalse {
$xtrace
}
function isset {
[[ -v "$1" ]]
}
#function isset {
# [[ -v "$1" ]]
#}
#
# Control Functions
# =================
......@@ -1839,17 +1839,18 @@ function cp_it {
#
# http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
function export_proxy_variables {
if isset http_proxy ; then
export http_proxy=$http_proxy
fi
if isset https_proxy ; then
export https_proxy=$https_proxy
fi
if isset no_proxy ; then
export no_proxy=$no_proxy
fi
}
# Unused function
#function export_proxy_variables {
# if isset http_proxy ; then
# export http_proxy=$http_proxy
# fi
# if isset https_proxy ; then
# export https_proxy=$https_proxy
# fi
# if isset no_proxy ; then
# export no_proxy=$no_proxy
# fi
#}
# Returns true if the directory is on a filesystem mounted via NFS.
function is_nfs_directory {
......
#!/bin/bash
#### Setup Ansible and load few libraries #####
BASE_SETUP_SCRIPT_MD5={{ base_setup.stat.md5 }}
wget --no-check-certificate https://deploy.nexedi.cn/base-setup -O /tmp/base-setup
if [ "`md5sum /tmp/base-setup | cut -f1 -d\ `" != "$BASE_SETUP_SCRIPT_MD5" ]; then
echo "ERROR: base-setup has wrong md5 `md5sum /tmp/base-setup | cut -f1 -d\ ` != $BASE_SETUP_SCRIPT_MD5"
exit 1
fi
source /tmp/base-setup
{{ lookup('file', 'roles/install-script/files/function-common') }}
export PATH=$PATH:/usr/local/bin:/usr/local/sbin
# Include Additional Functions
function download_playbook {
if [ ! -f /etc/opt/slapcache.cfg ]; then
......@@ -37,7 +39,7 @@ fi
# Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``export FORCE=yes``
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|rhel7) ]]; then
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|rhel7|rhel6) ]]; then
echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes"
......@@ -50,7 +52,7 @@ if [[ $DISTRO == "wheezy" ]]; then
fi
if is_fedora && [[ $DISTRO == "rhel7" ]]; then
# RHEL requires EPEL for many Open Stack dependencies
# RHEL requires EPEL for many Ansible dependencies
# NOTE: We always remove and install latest -- some environments
# use snapshot images, and if EPEL version updates they break
......@@ -89,6 +91,31 @@ EOF
fi
if is_fedora && [[ $DISTRO == "rhel6" ]]; then
if sudo yum repolist enabled epel | grep -q 'epel'; then
uninstall_package epel-release || true
fi
cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
[epel-bootstrap]
name=Bootstrap EPEL
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-6&arch=\$basearch
failovermethod=priority
enabled=0
gpgcheck=0
EOF
yum-config-manager --enable epel-bootstrap
yum_install epel-release || \
die $LINENO "Error installing EPEL repo, cannot continue"
# EPEL rpm has installed it's version
rm -f /etc/yum.repos.d/epel-bootstrap.repo
is_package_installed yum-utils || install_package yum-utils
yum-config-manager --enable rhel-6-server-optional-rpms
is_package_installed libselinux-python || install_package libselinux-python
fi
is_package_installed ansible || install_package ansible
is_package_installed python-setuptools || install_package python-setuptools
......@@ -97,5 +124,4 @@ if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
is_package_installed python-pycurl || install_package python-pycurl
fi
ansible localhost -m easy_install -a name=slapcache --connection=local
---
- name: Install ntp
apt: name=ntp state=latest
when: ansible_os_family == "Debian"
......@@ -10,3 +9,8 @@
- name: ensure ntp is runing
service: name=ntp state=running enabled=yes
when: ansible_os_family == "Debian"
- name: ensure ntp is runing (RedHat)
service: name=ntpd state=running enabled=yes
when: ansible_os_family == "RedHat"
\ No newline at end of file
......@@ -7,7 +7,8 @@ import glob
import time
import getopt
import sqlite3
from urllib2 import urlopen
import ssl
import urllib2
from xml.dom import minidom
import json
......@@ -20,15 +21,21 @@ def discover_software():
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT DISTINCT software_release FROM partition11")
return [row[0] for row in qry]
return [row[0] for row in qry if row[0]]
def get_connection_information(software_release):
conn = sqlite3.connect("/opt/slapos/slapproxy.db")
cur = conn.cursor()
qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=?", (software_release,) )
xml = None
for row in qry:
xml = str(row[0])
break
if xml is None:
print software_release
return (None, None)
instance = minidom.parseString(xml)
try:
......@@ -90,13 +97,51 @@ def status(software_release):
fmt_date() + ".log for details"
sys.exit(2)
ipv6 = None
# check if the services are actually running (run slapos node and parse output)
if pw is None:
zope_ip = "https://" + zope_ip[zope_ip.index("@")+1:]
r1 = urlopen(zope_ip)
original_zope_ip = zope_ip
if "[" in zope_ip and "]" in zope_ip:
ipv6 = zope_ip[zope_ip.index("[")+1:zope_ip.index("]")]
with open("/etc/hosts", "ra+") as f:
if " erp5-instance" not in f.read():
f.write("\n%s erp5-instance\n" % ipv6)
zope_ip = zope_ip.replace("[" + ipv6 + "]", "erp5-instance")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
try:
r1 = urllib2.urlopen(zope_ip, context=ctx)
except urllib2.URLError, e:
print "At least one of your services isn't running! Check with slapos node"
print "restart a service with slapos node restart slappart:service"
print ""
print "DEBUG information: %s" % e
sys.exit(2)
if r1.getcode() != 200:
print "At least one of your services isn't running! Check with slapos node"
print "restart a service with slapos node restart slappart:service"
sys.exit(2)
if ipv6:
print ""
print "The URL above may require extra configuration if you want to access it"
print "from another machine. You can install an apache locally and include the"
print "the follow rewrite rule (http version):"
print """
RewriteRule ^/(.*) %s/VirtualHostBase/http/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
or (https version):
RewriteRule ^/(.*) %s/VirtualHostBase/https/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
""" % (original_zope_ip, original_zope_ip)
def info(software_release):
if get_build_status():
......
---
- name: create partition script
template: src={{ request_instance_template }} dest=/tmp/playbook-request-{{ playbook_name }} mode=700
- name: Supply and Request ERP5 Cluster
shell: cat /tmp/playbook-request-{{ playbook_name }} | slapos console
- name: create erp5-show
copy: src=erp5-show dest=/usr/local/bin/erp5-show mode=755
- name: Add startup script
template: src=start-script.j2 dest=/usr/local/bin/{{ playbook_name }}-startup mode=755
- name: Add to rc.local
lineinfile:
dest=/etc/rc.local insertbefore=BOF
line='bash /usr/local/bin/{{ playbook_name }}-startup &'
state=present
- name: Get slapos.playbook directory name
shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
register: tmp_dir
- name: Check if /opt/slapos.playbook already exists
stat: path=/opt/slapos.playbook/
register: playbook_state
- name: Copy slapos.playbook
copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
when: playbook_state.stat.exists == False
import json
software_url = 'https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1/software/erp5/software.cfg'
software_url = '{{ software_release_url }}'
# Choose a SlapOS Node
# If you are deploying SlapOS Master with Webrunner, then computer_id is 'slaprunner'
computer_id = 'local_computer'
supply(software_url, computer_id)
parameter_dict = {
"timezone": "UTC",
"site-id": "erp5",
......@@ -33,7 +36,7 @@ parameter_dict = {
}
# Choose a title
title = "instance-of-erp5-cluster"
title = "instance-of-{{ playbook_name }}"
request(software_url,
title,
......
import json
# copied from the request-erp5-cluster script, only changed the url to point to wendelin
software_url = 'https://lab.nexedi.cn/nexedi/slapos/raw/1.0/software/wendelin/software.cfg'
# Choose a SlapOS Node
# If you are deploying SlapOS Master with Webrunner, then computer_id is 'slaprunner'
software_url = '{{ software_release_url }}'
computer_id = 'local_computer'
parameter_dict = {
}
# Choose a title
title = "instance-of-wendelin"
title = "instance-of-{{ playbook_name }}"
supply(software_url, computer_id)
request(software_url,
title,
......
#!/bin/bash
# Reruns the ansible playbook, does nothing else
cd /opt/slapos.playbook/playbook/
ansible-playbook {{ startup_playbook_id }} -i hosts --connection=local
---
- include: remove-slapos-package.yml
- name: a play that runs entirely on the ansible host
hosts: 127.0.0.1
connection: local
tasks:
- name: Remove slapos.package setup and configuration
shell: rm -rf /usr/local/lib/python2.*/dist-packages/slapos.package*
- name: Remove slapos.package setup and configuration
file: path={{ item }} state=absent
with_items:
- /usr/local/bin/slappkg-discover
- /usr/local/bin/slappkg-conf
- /usr/local/bin/slappkg-update
- /usr/local/bin/slappkg-update-raw
- /usr/local/bin/slappkg-upload-key
- /etc/opt/update.cfg
#!/usr/bin/python2.7
import socket
import errno
import cgi
import os
from BaseHTTPServer import BaseHTTPRequestHandler
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
class MyHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/ip':
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write('Your IP address is %s' % self.client_address[0])
return
else:
return SimpleHTTPRequestHandler.do_GET(self)
def do_POST(self):
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={"REQUEST_METHOD": "POST"}
)
lw.write_log(form.list[0].value, form.list[1:])
self.send_response(200)
class HTTPServerV6(HTTPServer):
address_family = socket.AF_INET6
class logWriter():
def write_log(self, fp, contents):
print os.path.dirname(fp)
try:
os.makedirs(os.path.dirname(fp))
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
try:
with open(fp, 'a') as myfile:
for l in contents:
myfile.write(l.value)
except IOError as e:
print "Unable to open file" #Does not exist OR no read permissions
with open(fp, 'w') as myfile:
for l in contents:
myfile.write(l.value)
print "Finished writing"
if __name__ == '__main__':
lw = logWriter()
server = HTTPServerV6(('::', 9000), MyHandler)
print 'Starting server, use <Ctrl-C> to stop'
server.serve_forever()
#!/usr/bin/python2.7
import os
import sys
import time
from glob import glob
import requests
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
class MyHandler(FileSystemEventHandler):
def on_created(self, event):
if ".completed" in event.file:
print "partition completed!"
upload_file(event.file, fmt_date() + ": partition completed")
def fmt_date():
return time.strftime("[ %Y%m%d %H:%M:%S ]")
def dir_date():
return time.strftime("%Y%m%d/")
class uploader():
url = 'http://[2001:67c:1254:e:a9::bafb]:9000/post'
# takes a file name and contents to append. Generates server file_path (where it should be saved on server)
def upload_file(self, file_name, contents):
values = {'file_path': dir_date() + file_name, 'contents': contents}
r = requests.post(self.url, data=values)
def check_startup():
pid = str(os.getpid())
pidfile = "/tmp/slaplogger.pid"
if os.path.isfile(pidfile):
print "%s already exists, exiting" % pidfile
sys.exit()
else:
file(pidfile, 'w').write(pid)
os.unlink(pidfile)
def scan_files(files):
try:
with open("/opt/slapos/lines", 'r') as myfile:
software_line = myfile.readlines()
except:
software_line = [0] * len(files)
with open("/opt/slapos/lines", 'w'):
pass
place = 0
for f in files:
print software_line[place]
print software_line
with open(f, 'r') as myfile:
new_lines = myfile.readlines()[software_line[place]:]
with open("/opt/slapos/lines", 'a') as myfile:
myfile.write(str(software_line[place]) + str(len(new_lines)))
print "Uploading " + f + "..."
c.upload_file(os.path.basename(f), new_lines)
print f + " finished uploading!"
place += 1
print "Files uploaded"
if __name__ == "__main__":
check_startup()
c = uploader()
files = ["/opt/slapos/log/slapos-node-software.log", "/opt/slapos/log/slapos-node-format.log", "/opt/slapos/log/slapos-node-instance.log"]
# first check if .completed exists in any partitions
paths = ["/opt/slapgrid/slappart" + p + ".completed" for p in glob('/opt/slapgrid/*')]
for p in paths:
try:
f = open(p, 'r')
upload_file(p, fmt_date() + ": partition completed")
except IOError as e:
pass
event_handler = MyHandler()
observer = Observer()
# check if .completed exists or not, and write it to the server log
observer.schedule(event_handler, path='/opt/slapgrid/', recursive=True)
observer.start()
try:
while True:
scan.files(files)
except KeyboardInterrupt:
observer.stop()
observer.join()
---
- name: download software_release name
get_url: url=https://lab.nexedi.cn/krruzic/test-repository/raw/master/software
dest=/tmp/playbook_software_name
- name: create log-uploader
copy: src=log-uploader dest=/usr/local/bin/log-uploader mode=755
- name: add log-uploader to cron
cron: name="Ensure log-uploader is running"
minute="*/6"
job="usr/local/bin/log-uploader"
- name: run ansible playbook
shell: ansible-playbook "{{ lookup('file', '/tmp/playbook_software_name') }}" -i hosts
# install dependencies
- easy_install: name=pip
- pip: name=requests
- pip: name=watchdog
......@@ -41,7 +41,7 @@
when: hostname_file.stat.exists == True
- name: setting cluster
get_url: url="{{ lookup('file', '/tmp/cluster.hash') }}/hosts" dest=/tmp/hosts mode=666 validate_certs=no
shell: wget --no-check-certificate "{{ lookup('file', '/tmp/cluster.hash') }}/hosts" -O /tmp/hosts
when: cluster_hash.stat.exists == True
- name: stat /tmp/hosts
......
#!/bin/bash
# Reruns the ansible playbook, does nothing else
PLAYBOOK_ROOT=/opt/slapos.playbook/
PLAYBOOK_FILE=wendelin-standalone.yml
cd $PLAYBOOK_ROOT # cd into the playbook directory
ansible-playbook $PLAYBOOK_FILE -i hosts --connection=local
---
- name: Supply wendelin software release
shell: slapos supply {{ software_release_url }} local_computer
- name: create partition script
copy: src=request-wendelin dest=/tmp/playbook-request-wendelin mode=700
- name: Request Wendelin instance
shell: cat /tmp/playbook-request-wendelin | slapos console
- name: Add startup script
copy: src=wendelin-startup dest=/usr/local/bin/wendelin-startup mode=755
- name: Add to rc.local
lineinfile:
dest=/etc/rc.local insertbefore=BOF
line='bash /usr/local/bin/wendelin-startup &'
state=present
- name: Get slapos.playbook directory name
shell: cd /tmp/tmpplaybookwendelin-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
register: tmp_dir
- name: Check if /opt/slapos.playbook already exists
stat: path=/opt/slapos.playbook/
register: playbook_state
- name: Copy slapos.playbook
copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
when: playbook_state.stat.exists == False
- name: a play that runs entirely on the ansible host
hosts: 127.0.0.1
connection: local
roles:
- test-suite
......@@ -3,8 +3,10 @@
connection: local
vars:
software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/1.0/software/wendelin/software.cfg
software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/1.0.3/software/wendelin/software.cfg
startup_playbook_id: wendelin-standalone.yml
playbook_name: wendelin
request_instance_template: request-wendelin.j2
roles:
- wendelin-standalone
- standalone-shared
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!