Add BonjourGrid recipe to slapos.cookbook

setup.py

@@ -50,6 +50,8 @@ setup(name=name,
 	        'boinc = slapos.recipe.boinc:Recipe',
           'boinc.app = slapos.recipe.boinc:App',
           'boinc.client = slapos.recipe.boinc:Client',
+          'bonjourgrid = slapos.recipe.bonjourgrid:Recipe',
+          'bonjourgrid.client = slapos.recipe.bonjourgrid:Client',
           'certificate_authority.request = slapos.recipe.certificate_authority:Request',
           'certificate_authority = slapos.recipe.certificate_authority:Recipe',
           'check_port_listening = slapos.recipe.check_port_listening:Recipe',

slapos/recipe/bonjourgrid/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import os
+import subprocess
+import pwd
+import signal
+import zc.buildout
+
+class Recipe(GenericBaseRecipe):
+  """Deploy a fully operational BonjourGrid Master architecture."""
+
+
+  def install(self):
+    path_list = []
+    condor_wrapper = self.options['condor-wrapper'].strip()
+    boinc_wrapper = self.options['boinc-wrapper'].strip()
+    startCondor = startBoinc = False
+    condor_wrapper_list = []
+    boinc_wrapper_list = []
+    for file in os.listdir(condor_wrapper):
+      condor_wrapper_list.append(os.path.join(condor_wrapper, file))
+    for file in os.listdir(boinc_wrapper):
+      boinc_wrapper_list.append(os.path.join(boinc_wrapper, file))
+
+    type = self.options['type'].strip()
+    if type == "condor":
+      startCondor = True
+      hostname = self.options['condor_host'].strip()
+      url = "http://%s" % hostname
+    if type == "boinc":
+      import socket
+      hostname = socket.gethostname()
+      url = self.options['url-boinc'].strip()
+      startBoinc = True
+
+    #Generate BOINC/Condor launcher script
+    grid_wrapper = self.options['boinc_condor_wrapper'].strip()
+    parameters = dict(startCondor=startCondor, startBoinc=startBoinc,
+                      bg_base=self.options['work_dir'].strip(),
+                      condor_wrapper_list=condor_wrapper_list,
+                      boinc_wrapper_list=boinc_wrapper_list)
+    bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
+        '%s.configure.launchScript' % __name__,
+        parameters
+    )
+    path_list.append(bonjourGrid_wrapper)
+
+    #Generate wrapper for BonjourGrid Master
+    bonjourgrid_master = self.options['master_script'].strip()
+    python = self.options['python-bin'].strip()
+    bg_wrapper = self.options['wrapper'].strip()
+    log = self.options['log_file'].strip()
+    pid_file = self.options['pid_file'].strip()
+    wrapper = self.createPythonScript(bg_wrapper,
+        'slapos.recipe.librecipe.execute.execute',
+        ([python, bonjourgrid_master, '--log_file', log,
+          '--pid_file', pid_file,
+          '--master_wrapper', grid_wrapper,
+          '--directory', self.options['work_dir'].strip(),
+          '--install_directory', self.options['install_dir'].strip(),
+          '--server', self.options['redis-url'].strip(),
+          '--port', self.options['redis-port'].strip(),
+          '--num_workers', self.options['nworkers'].strip(),
+        ])
+    )
+    path_list.append(wrapper)
+
+
+    #generate Computer information file
+    config_info_file = os.path.join(self.options['work_dir'].strip(),
+                                'machineinfo.sh')
+    config_info = self.createFile(config_info_file,
+      self.substituteTemplate(self.getTemplateFilename('machineinfo.sh.in'),
+      dict(ip_address=self.options['ipv6'].strip(),
+            hostname=hostname, url=url,
+            middleware=type)))
+    os.chmod(config_info_file, 0744)
+    path_list.append(config_info)
+
+class Client(GenericBaseRecipe):
+
+  def install(self):
+    path_list = []
+    boinc_script = self.options['boinc_script'].strip()
+    condor_script = self.options['condor_script'].strip()
+
+    #Generate wrapper for BonjourGrid Worker
+    bonjourgrid_client = self.options['client_script'].strip()
+    python = self.options['python-bin'].strip()
+    bg_wrapper = self.options['wrapper'].strip()
+    log = self.options['log_file'].strip()
+    pid_file = self.options['pid_file'].strip()
+    wrapper = self.createPythonScript(bg_wrapper,
+        'slapos.recipe.librecipe.execute.execute',
+        ([python, bonjourgrid_client, '--log_file', log,
+          '--pid_file', pid_file,
+          '--boinc_wrapper', boinc_script,
+          '--condor_wrapper', condor_script,
+          '--directory', self.options['work_dir'].strip(),
+          '--install_directory', self.options['install_dir'].strip(),
+          '--server', self.options['redis-url'].strip(),
+          '--port', self.options['redis-port'].strip(),
+        ])
+    )
+    path_list.append(wrapper)
+
+    #generate BOINC and Condor configure script for bonjourgrid
+    boinc_wrapper = self.createPythonScript(boinc_script,
+        '%s.boinc.runBoinc' % __name__,
+        dict(ipv6=self.options['ipv6'].strip(),
+            email=self.options['email'].strip(),
+            boinc_wrapper=self.options['boinc_wrapper'].strip(),
+            boinc_cmd=self.options['boinc_cmd'].strip(),
+            boinc_rpc_port=self.options['boinc_rpc_port'],
+            boinc_install_dir=self.options['boinc_install_dir'].strip(),
+            boinc_passwd=self.options['boinc_passwd'].strip(),
+            account_name=self.options['account_name'].strip(),
+            account_passwd=self.options['account_passwd'].strip(),
+        )
+    )
+    path_list.append(boinc_wrapper)
+    condor_wrapper = self.createPythonScript(condor_script,
+        '%s.condor.runCondor' % __name__,
+        dict(ipv6=self.options['ipv6'].strip(),
+            condor_host=self.options['condor_host'].strip(),
+            collector_name=self.options['collector_name'].strip(),
+            condor_wrapper=self.options['condor_wrapper'].strip(),
+            condor_config=self.options['condor_config'].strip(),
+            condor_config_local=self.options['condor_config_local'].strip(),
+        )
+    )
+    path_list.append(condor_wrapper)
+

slapos/recipe/bonjourgrid/boinc.py

+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import re
+import subprocess
+import time
+import signal
+
+def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
+  process = subprocess.Popen(launch_args, stdout=stdout,
+              stderr=subprocess.STDOUT, env=env,
+              cwd=cwd)
+  result = process.communicate()[0]
+  if process.returncode is None or process.returncode != 0:
+    raise NameError("Failed to execute executable.\nThe error was: %s" % result)
+
+def joinProject(args, base_cmd):
+  """Finish BOINC Client configuration with create account and attach project"""
+  project_args = base_cmd + ['--project_attach', args['project_url'],
+                      args['key']]
+  startProcess(project_args, cwd=args['boinc_install_dir'])
+
+
+def createAccount(config, base_cmd):
+  """Connect to BOINC Master and create an account
+  """
+  account_args = base_cmd + ['--create_account', config['project_url'],
+                             config['email'], config['account_passwd'],
+                             config['account_name']]
+  startProcess(account_args, cwd=config['boinc_install_dir'])
+  account_file = os.path.join(config['boinc_install_dir'], 'create_account.xml')
+  key = re.search("<authenticator>([\w\d\._]+)</authenticator>",
+                open(account_file, 'r').read()).group(1)
+  return key
+
+def runBoinc(config):
+  if len(sys.argv) < 2:
+    print "Argument Error: uses %s project_url" % sys.argv[0]
+    exit(1)
+  if type(config) == type(""):
+    print "Error: bonjourgrid.xml parsing error, file not exist or corrupted"
+    exit(1)
+  #XXX Using define values here for Boinc Master URL
+  config['project_url'] = sys.argv[1]
+
+  #launch Boinc Client
+  boinc = subprocess.Popen([config['boinc_wrapper']],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.STDOUT)
+
+  #Retrieve ipv4 using by Boinc-client in client-configuration
+  client_config = os.path.join(config['boinc_install_dir'], 'client_state.xml')
+  while not os.path.exists(client_config):
+    time.sleep(5)
+    print "Search for file '%r'..." % client_config
+  time.sleep(10)
+  try:
+    #Scan client state xml to find client ipv4 adress
+    host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
+                  open(client_config, 'r').read()).group(1)
+    base_cmd = [config['boinc_cmd'], '--host',
+                host + ":" + str(config['boinc_rpc_port']),
+                '--passwd', config['boinc_passwd']]
+  
+    #Create Account for current instance on BOINC master
+    print "Create account for current client..."
+    key = createAccount(config, base_cmd)
+    config['key'] = key
+    print "Done. The account key is %s" % key
+  
+    #Attach project to Boinc Master
+    print "Attach client to Boinc Master at %s " % config['project_url']
+    try:
+      joinProject(config, base_cmd)
+    except Exception, e:
+      print e
+    print "Done! Waiting for Boinc Client now..."
+  except Exception, e:
+    #An error occure!!!
+    os.kill(boinc.pid, signal.SIGTERM)
+    print e
+  #wait for Boinc client execution
+  boinc.wait()

slapos/recipe/bonjourgrid/condor.py

+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import re
+import subprocess
+import time
+
+def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
+  process = subprocess.Popen(launch_args, stdout=stdout,
+              stderr=subprocess.STDOUT, env=env,
+              cwd=cwd)
+  result = process.communicate()[0]
+  if process.returncode is None or process.returncode != 0:
+    raise NameError("Failed to execute executable.\nThe error was: %s" % result)
+
+def runCondor(config):
+  if len(sys.argv) < 2:
+    print "Argument Error: uses %s hostname projectname" % sys.argv[0]
+    exit(1)

slapos/recipe/bonjourgrid/configure.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import subprocess
+import time
+
+def runProcess(args, file):
+  filename = file.split('/')[-1:][0]
+  stdout_file = open(os.path.join(args['bg_log'], filename + ".log"), 'w')
+  process = subprocess.Popen(file, stdout=stdout_file,
+          stderr=subprocess.STDOUT)
+  fp = open(os.path.join(args['bg_pid'], filename + '.pid'), 'w')
+  fp.write(str(process.pid))
+  fp.close()
+  return process.pid
+
+def launchScript(args):
+  print "Sleep for a few second..."
+  time.sleep(10)
+  pid_list = []
+  bg_pid = os.path.join(args['bg_base'], 'pid')
+  bg_log = os.path.join(args['bg_base'], 'log')
+  args['bg_pid'] = bg_pid
+  args['bg_log'] = bg_log
+  if not os.path.exists(bg_pid):
+    os.mkdir(bg_pid)
+  if not os.path.exists(bg_log):
+    os.mkdir(bg_log)
+
+  #launch all Condor wrapper
+  if args['startCondor']:
+    for file in args['condor_wrapper_list']:
+      pid_list.append(runProcess(args, file))
+
+  #Launch all BOINC wrapper
+  if args['startBoinc']:
+    for file in args['boinc_wrapper_list']:
+      pid_list.append(runProcess(args, file))
+
+  for pid in pid_list:
+    print "Parent waiting for process child: %s " % pid
+    result = os.waitpid(pid, 0)
+    print "Done...", result

slapos/recipe/bonjourgrid/template/machineinfo.sh.in

+# This script generates a file (with the name `hostname`) of the characteristics of the machine
+
+Mhz=`grep -i Mhz /proc/cpuinfo | cut -d : -f2 | head -1 | cut -f2 -d " "`
+mem=`grep -i MemTotal /proc/meminfo | cut -f2 -d :`
+Ram=`echo $mem | cut -f1 -d " "`
+Cpu=`uname -a | cut -f3 -d " "`
+H=%(hostname)s
+Nbproc=`grep processor /proc/cpuinfo |wc -l`
+Ip=%(ip_address)s
+status="in execution"
+secours="host1"
+middleware="%(middleware)s"
+url="%(url)s"
+
+c="{'"MHZ"':\"$Mhz\",'"Ram"':\"$Ram\",'"Cpu"':\"$Cpu\",'"HOST"':\"$H\",'"IP"':\"$Ip\",'"STATUS"':\"$status\",'"SECOURS"':\"$secours\",'"MIDDLEWARE"':\"$middleware\",'"URL"':\"$url\",}"
+
+filename=`hostname`
+echo $c > $filename