Merge branch 'master' into 'master'

Merge test-suite from test-repo, remove unneeded parts, support log upload and watch install progress

Adds log uploading to a slapos webrunner, makes sure ipv6 is installed so we can actually access it. Keeps track of changes to the install, and reports to the server if ".completed" exists (which should only happen if the software finished installing).

Doesn't do anything with the logs, and they're publicly accessible as long as you know the IP. Don't know where to go from here.

See merge request !2

playbook/roles/test-suite/files/log-server

+#!/usr/bin/python2.7
+import socket
+import errno
+import cgi
+import os
+from BaseHTTPServer import BaseHTTPRequestHandler
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+
+class MyHandler(BaseHTTPRequestHandler):
+
+  def do_GET(self):
+    if self.path == '/ip':
+      self.send_response(200)
+      self.send_header('Content-type', 'text/html')
+      self.end_headers()
+      self.wfile.write('Your IP address is %s' % self.client_address[0]) 
+      return    
+    else:
+      return SimpleHTTPRequestHandler.do_GET(self)
+
+  def do_POST(self):     
+    form = cgi.FieldStorage(
+      fp=self.rfile,
+      headers=self.headers,
+      environ={"REQUEST_METHOD": "POST"}
+    )
+
+    lw.write_log(form.list[0].value, form.list[1:])
+    self.send_response(200)
+
+class HTTPServerV6(HTTPServer):
+  address_family = socket.AF_INET6
+
+class logWriter():
+  def write_log(self, fp, contents):
+    print os.path.dirname(fp)
+    try:
+      os.makedirs(os.path.dirname(fp))
+    except OSError as exception:
+      if exception.errno != errno.EEXIST:
+        raise
+    try:
+      with open(fp, 'a') as myfile:
+        for l in contents:
+          myfile.write(l.value)
+    except IOError as e:
+      print "Unable to open file" #Does not exist OR no read permissions
+      with open(fp, 'w') as myfile:
+        for l in contents:
+          myfile.write(l.value)
+    print "Finished writing"
+
+if __name__ == '__main__':  
+  lw = logWriter()
+  server = HTTPServerV6(('::', 9000), MyHandler)
+  print 'Starting server, use <Ctrl-C> to stop'
+  server.serve_forever()

playbook/roles/test-suite/files/log-uploader

+#!/usr/bin/python2.7
+
+import os
+import sys
+import time
+from glob import glob
+import requests
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+class MyHandler(FileSystemEventHandler):
+  def on_created(self, event):
+    if ".completed" in event.file:
+      print "partition completed!"
+      upload_file(event.file, fmt_date() + ": partition completed")
+
+def fmt_date():
+  return time.strftime("[ %Y%m%d %H:%M:%S ]")
+
+def dir_date():
+  return time.strftime("%Y%m%d/")
+
+class uploader():
+  url = 'http://[2001:67c:1254:e:a9::bafb]:9000/post'
+  # takes a file name and contents to append. Generates server file_path (where it should be saved on server)
+  def upload_file(self, file_name, contents):
+    values = {'file_path': dir_date() + file_name, 'contents': contents}
+    r = requests.post(self.url, data=values)
+
+def check_startup():
+  pid = str(os.getpid())
+  pidfile = "/tmp/slaplogger.pid"
+
+  if os.path.isfile(pidfile):
+    print "%s already exists, exiting" % pidfile
+    sys.exit()
+  else:
+    file(pidfile, 'w').write(pid)
+  os.unlink(pidfile)
+
+def scan_files(files):
+  try:  
+    with open("/opt/slapos/lines", 'r') as myfile:
+      software_line = myfile.readlines()
+  except:
+    software_line = [0] * len(files)
+  with open("/opt/slapos/lines", 'w'):
+    pass  
+  place = 0
+
+  for f in files:
+    print software_line[place]
+    print software_line
+    with open(f, 'r') as myfile:
+      new_lines = myfile.readlines()[software_line[place]:]
+    with open("/opt/slapos/lines", 'a') as myfile:
+      myfile.write(str(software_line[place]) + str(len(new_lines)))
+    print "Uploading " + f + "..."
+    c.upload_file(os.path.basename(f), new_lines)
+    print f + " finished uploading!"
+    place += 1
+  print "Files uploaded"
+
+
+if __name__ == "__main__":
+  check_startup()
+  c = uploader()
+  files = ["/opt/slapos/log/slapos-node-software.log", "/opt/slapos/log/slapos-node-format.log", "/opt/slapos/log/slapos-node-instance.log"]
+
+  # first check if .completed exists in any partitions
+  paths = ["/opt/slapgrid/slappart" + p + ".completed" for p in glob('/opt/slapgrid/*')]
+  for p in paths: 
+    try:
+      f = open(p, 'r')
+      upload_file(p, fmt_date() + ": partition completed")    
+    except IOError as e:
+      pass
+
+  event_handler = MyHandler()
+  observer = Observer()
+
+
+  # check if .completed exists or not, and write it to the server log
+  observer.schedule(event_handler, path='/opt/slapgrid/', recursive=True)
+  observer.start()
+
+  try:
+    while True:
+      scan.files(files)
+  except KeyboardInterrupt:
+    observer.stop()
+  observer.join()

playbook/roles/test-suite/tasks/main.yml

+---
+  - name: download software_release name
+    get_url: url=https://lab.nexedi.cn/krruzic/test-repository/raw/master/software
+             dest=/tmp/playbook_software_name
+
+  - name: create log-uploader
+    copy: src=log-uploader dest=/usr/local/bin/log-uploader mode=755
+
+  - name: add log-uploader to cron
+    cron: name="Ensure log-uploader is running" 
+          minute="*/6"
+          job="usr/local/bin/log-uploader"
+
+  - name: run ansible playbook
+    shell: ansible-playbook "{{ lookup('file', '/tmp/playbook_software_name') }}" -i hosts
+
+  # install dependencies
+  - easy_install: name=pip
+  - pip: name=requests
+  - pip: name=watchdog

playbook/test-suite.yml

+- name: a play that runs entirely on the ansible host
+  hosts: 127.0.0.1
+  connection: local
+
+  roles:
+    - re6st # make sure we have ipv6
+    - test-suite