Add Stork Component and Recipe (with Walid SaaD)

component/globus/buildout.cfg

+[buildout]
+parts =
+  globus
+
+extends =
+  ../openssl/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../perl/buildout.cfg
+  ../libaio/buildout.cfg
+  ../libtool/buildout.cfg
+  ../zlib/buildout.cfg
+
+
+[globus]
+recipe = slapos.recipe.cmmi
+url = http://toolkit.globus.org/ftppub/gt5/5.2/5.2.5/installers/src/gt5.2.5-all-source-installer.tar.gz
+md5sum = 10ecf1cdb3c4381cb4c1534f393d263f
+
+environment =
+  PATH=${pkgconfig:location}/bin:${perl:location}/bin/:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
+  CPPFLAGS=-I${libtool:location}/include -I${zlib:location}/include -I${openssl:location}/include
+  LDFLAGS=-L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib
\ No newline at end of file

component/stork/buildout.cfg

+[buildout]
+
+parts =
+  stork
+
+extends =
+  ../dash/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../perl/buildout.cfg
+  ../java/buildout.cfg
+  ../openssl/buildout.cfg
+  ../globus/buildout.cfg
+  ../../stack/slapos.cfg
+
+[download-stork]
+recipe = hexagonit.recipe.download
+url = http://sourceforge.net/projects/bonjour-grid/files/stork-package.tar.gz
+strip-top-level-dir = true
+md5sum = e228a7e0176c3af5167b83273b39f60a
+
+[download-ksh]
+recipe = hexagonit.recipe.download
+download-only = true
+url= http://garr.dl.sourceforge.net/project/bonjour-grid/ksh.2012-08-06.linux.i386-64
+filename = ksh.2012-08-06.linux.i386-64
+md5sum = 6b3824d6aacac7cfe7067bc8efb7e3dc
+
+# Compile now  stork
+[stork]
+recipe = plone.recipe.command
+on_install = true
+on_update = true
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+ksh = ${download-ksh:location}/${download-ksh:filename}
+stop-on-error = true
+command = cd ${download-stork:location}  && \
+    chmod 744 ${:ksh} && ${:ksh} stork-install.ksh ${:location} && \
+    echo "Finished."
+environment =
+  LD_LIBRARY_PATH=${libexpat:location}/lib:${openssl:location}/lib:${zlib:location}/lib:${globus:location}/lib
+  PATH=${perl:location}/bin:${java:location}/bin:${openssl:location}/bin:%(PATH)s

setup.py

@@ -194,6 +194,8 @@ setup(name=name,
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
           'stunnel = slapos.recipe.stunnel:Recipe',
+          'stork = slapos.recipe.stork:Recipe',
+          'stork.submit = slapos.recipe.stork:AppSubmit',
           'symbolic.link = slapos.recipe.symbolic_link:Recipe',
           'tidstorage = slapos.recipe.tidstorage:Recipe',
           'trac = slapos.recipe.trac:Recipe',

slapos/recipe/stork/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import os
+import json
+
+class Recipe(GenericBaseRecipe):
+  """Deploy a fully operational condor architecture."""
+
+  def __init__(self, buildout, name, options):
+    self.environ = {}
+    self.role = ''
+    environment_section = options.get('environment-section', '').strip()
+    if environment_section and environment_section in buildout:
+      # Use environment variables from the designated config section.
+      self.environ.update(buildout[environment_section])
+    for variable in options.get('environment', '').splitlines():
+      if variable.strip():
+        try:
+          key, value = variable.split('=', 1)
+          self.environ[key.strip()] = value
+        except ValueError:
+          raise zc.buildout.UserError('Invalid environment variable definition: %s', variable)
+    # Extrapolate the environment variables using values from the current
+    # environment.
+    for key in self.environ:
+      self.environ[key] = self.environ[key] % os.environ
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def _options(self, options):
+    #Path of stork compiled package
+    self.package = options['package'].strip()
+    self.rootdir = options['rootdirectory'].strip()
+    #Other stork dependances
+    self.javabin = options['java-bin'].strip()
+    self.dash = options['dash'].strip()
+    #Directory to deploy stork
+    self.prefix = options['rootdirectory'].strip()
+    self.wrapper_bin = options['bin'].strip()
+    self.wrapper_sbin = options['sbin'].strip()
+    self.wrapper_log= options['log'].strip()
+    self.wrapper_tmp= options['tmp'].strip()
+    self.ipv4 = options['ipv4'].strip()
+    self.stork_host = options['stork_host'].strip()
+    self.stork_port = options['stork_port'].strip()
+  def install(self):
+    path_list = []
+    #get UID and GID for current slapuser
+    stat_info = os.stat(self.rootdir)
+    slapuser = str(stat_info.st_uid)+"."+str(stat_info.st_gid)
+    domain_name = 'slapos%s.com' % stat_info.st_uid
+
+
+    #Generate stork_mabroukconfig file
+    stork_config = os.path.join(self.rootdir, 'etc/stork_config')
+    stork_configure = dict(stork_host=self.stork_host, releasedir=self.prefix,
+                  storkpackage=self.package,
+                  slapuser=slapuser, ipv4=self.ipv4,
+                  port=self.stork_port)
+    destination = os.path.join(stork_config)
+     # case 1: client and server in the same instance
+    if (self.options['machine-role'] == "client") and (self.options['stork_server']=='local'):
+     config = self.createFile(destination,
+      self.substituteTemplate(self.getTemplateFilename('stork_config.generic'),
+      stork_configure))
+     path_list.append(config)
+    #case 2: client and server in different instances (the server may b a remote host)
+    else:
+     config = self.createFile(destination,
+      self.substituteTemplate(self.getTemplateFilename('remote_stork_config.generic'),
+      stork_configure))
+     path_list.append(config)
+
+
+    #create stork binary launcher for slapos
+    if not os.path.exists(self.wrapper_bin):
+      os.makedirs(self.wrapper_bin, int('0700', 8))
+    if not os.path.exists(self.wrapper_sbin):
+      os.makedirs(self.wrapper_sbin, int('0700', 8))
+    if not os.path.exists(self.wrapper_log):
+      os.makedirs(self.wrapper_log, int('0700', 8))
+    if not os.path.exists(self.wrapper_tmp):
+      os.makedirs(self.wrapper_tmp, int('0700', 8))
+    #generate script for each file in prefix/bin
+    for binary in os.listdir(self.package+'/bin'):
+      wrapper_location = os.path.join(self.wrapper_bin, binary)
+      current_exe = os.path.join(self.package, 'bin', binary)
+      wrapper = open(wrapper_location, 'w')
+      content = """#!%s
+      export LD_LIBRARY_PATH=%s
+      export PATH=%s
+      export STORK_CONFIG=%s
+      export STORK_HOME=%s
+      export STORK_IDS=%s
+      export HOSTNAME=%s
+      exec %s $*""" % (self.dash,
+              self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
+              stork_config, self.prefix, slapuser,
+              self.environ['HOSTNAME'], current_exe)
+      wrapper.write(content)
+      wrapper.close()
+      path_list.append(wrapper_location)
+      os.chmod(wrapper_location, 0700)
+
+    #generate script for each file in prefix/sbin
+    for binary in os.listdir(self.package+'/sbin'):
+      wrapper_location = os.path.join(self.wrapper_sbin, binary)
+      current_exe = os.path.join(self.package, 'sbin', binary)
+      wrapper = open(wrapper_location, 'w')
+      content = """#!%s
+      export LD_LIBRARY_PATH=%s
+      export PATH=%s
+      export STORK_CONFIG=%s
+      export STORK_HOME=%s
+      export STORK_IDS=%s
+      export HOSTNAME=%s
+      exec %s $*""" % (self.dash,
+              self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
+              stork_config, self.prefix, slapuser,
+              self.environ['HOSTNAME'], current_exe)
+      wrapper.write(content)
+      wrapper.close()
+      path_list.append(wrapper_location)
+      os.chmod(wrapper_location, 0700)
+
+    #generate script for start stork
+    if self.options['stork_server']=='local':
+     start_bin = os.path.join(self.wrapper_sbin, 'stork_server')
+     wrapper = self.createPythonScript(self.options['wrapper-path'].strip(),
+        '%s.configure.storkStart' % __name__,
+        dict(start_bin=start_bin,
+            port=self.stork_port,
+            configfile=self.rootdir+'/etc/stork_config',
+            pid=self.options['pid']))
+     path_list.append(wrapper)
+    return path_list
+
+class AppSubmit(GenericBaseRecipe):
+  """Submit a stork job into an existing Stork server instance"""
+
+  def install(self):
+    path_list = []
+    download_dest_list = dict()
+    #check if curent stork instance is an stork server
+    if self.options['machine-role'].strip() == "server":
+      #raise Exception("Cannot submit a job to stork client instance")
+      print 'Cannot submit a job to stork client instance'
+      return path_list
+
+    #Setup directory
+    datadir = self.options['data-dir'].strip()
+    dest_url = self.options['dest_url'].strip()
+    source_url_list = self.options['json_src_url'].strip()
+    submitfile = os.path.join(datadir, 'submit-dap')
+    if self.options['dest_url']=='local':
+      dest_url = 'file:'+datadir
+    
+    stork_submit = os.path.join(self.options['bin'].strip(), 'stork_submit')
+    file_list = '{}'
+    if str(self.options.get('src_from_file')).lower() in ['y', 'yes', '1', 'true'] \
+            and os.path.exists(source_url_list):
+      with open(source_url_list, 'r') as file_source:
+        file_list = json.loads(file_source.read())
+    else:
+      file_list = json.loads(source_url_list)
+    with open(submitfile, 'w') as stork_file:
+      # XXX - Simply, we don't want to download file twice with stork,
+      #       skip file if it already exists at dest_url (only for file:path)
+      for filename in file_list:
+        if dest_url.startswith('file:'):
+          destination = os.path.join(dest_url.replace('file:', ''), filename)
+          if os.path.exists(destination+'_part') or os.path.exists(destination):
+            continue
+          download_dest_list[filename] = destination
+        stork_file.write("""[
+src_url="%s";
+dest_url="%s/%s_part";
+err = "%s.err";
+output = "%s.out";
+dap_type = "transfer";
+set_permission = "600" ;
+]
+
+""" % (file_list[filename], dest_url, filename, filename, filename)
+          )
+  
+    parameter = dict(submit=stork_submit,
+                    submit_file=submitfile,
+                    stork_server=self.options['ipv4'].strip(),
+                    server_port=self.options['stork_port'].strip(),
+                    datadir=datadir)
+    submit_job = self.createPythonScript(
+      self.options['wrapper-path'].strip(),
+      '%s.configure.submitJob' % __name__, parameter
+    )
+    path_list.append(submit_job)
+    
+    check_job = self.createPythonScript(
+      self.options['wrapper-check'].strip(),
+      '%s.configure.checkDownloadStatus' % __name__,
+      dict(dest_list=download_dest_list, cwd=self.options['log'])
+    )
+    path_list.append(check_job)
+    
+    return path_list
\ No newline at end of file

slapos/recipe/stork/configure.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import subprocess
+import time
+
+def submitJob(args):
+  """Run stork_submit (if needed) for job deployment"""
+  time.sleep(10)
+
+  # '-a', "log = out.log", '-a', "error = error.log",
+  server_info=args['stork_server']+':'+args['server_port']
+  launch_args = [args['submit'],server_info, args['submit_file']]
+  process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
+              stderr=subprocess.STDOUT, cwd=args['datadir'])
+  result = process.communicate()[0]
+  if process.returncode is None or process.returncode != 0:
+    print "Failed to execute stork_submit.\nThe error was: %s" % result
+
+def storkStart(args):
+    """Start Stork """
+    proc_args = [args['start_bin'],
+                '-p', args['port'],
+                '-c', args['configfile']
+              ]
+    
+    process = subprocess.Popen(proc_args, stderr=subprocess.STDOUT,
+                                stdout=subprocess.PIPE)
+    time.sleep(3)
+    print "\n\nStork Server started with pid %s " % process.pid
+    with open(args['pid'], 'w') as f:
+      f.write(str(process.pid))
+    result = process.communicate()[0]
+    if process.returncode is None or process.returncode != 0:
+      print "Failed to start Stork Server.\nThe error was: %s" % result
+
+def checkDownloadStatus(args):
+  "Check if all files have been downloaded! "
+  job_count = len(args['dest_list'])
+  os.chdir(args['cwd'])
+
+  while job_count > 0:
+    print "Check if %s files have been downloaded!" % job_count
+    for filename in args['dest_list']:
+      log_name = "%s.err" % filename
+      destination = args['dest_list'][filename]
+      if os.path.exists(destination):
+        job_count -= 1
+        continue
+      if os.path.exists(destination + '_part') and os.path.exists(log_name):
+        with open(log_name, 'r') as log_file:
+          if not log_file.readline().startswith('SUCCESS'):
+            with open(filename + '.out', 'r') as out_file:
+              if not out_file.readline().startswith('SUCCESS'):
+                continue
+        job_count -= 1
+        time.sleep(2)
+        os.rename(destination + '_part', destination)
+        print "File %s is correctly downloaded!" % destination
+      else:
+        print "ERROR!! Check if destination and log_files exist... %s" % destination
+    
+    time.sleep(5)
+    
\ No newline at end of file

slapos/recipe/stork/template/remote_stork_config.generic

+######################################################################
+##
+##  stork_config
+##
+######################################################################
+ENABLE_IPV6 = TRUE
+PORT = 9621
+NETWORK_INTERFACE = %(ipv4)s
+COLLECTOR_HOST =  %(ipv4)s:$(PORT)
+CONDOR_HOST =  %(ipv4)s
+
+RELEASE_DIR             =  %(releasedir)s
+##Stork startup arguments
+## Start Stork on a well-known port.  Uncomment to simplify connecting to a
+## remote Stork server.  Note: this interface may change in a future release.
+SBIN_DIR = %(releasedir)s/sbin
+STORK= $(SBIN_DIR)/stork_server
+
+ARGS = -p $(STORK_PORT) -f -Serverlog $(STORK_LOG_BASE)
+
+## HOST may specify an optional default remote Stork server and port.
+#HOST  = %(stork_host)s:$(PORT)
+
+HOST  = %(ipv4)s:$(PORT)
+## When Stork starts up, it can place its address (IP and port) into
+## a file. This way, tools running on the local machine don't
+## need an additional "-n host:port" command line option.  This
+## feature can be turned off by commenting out this setting.
+STORK_ADDRESS_FILE = $(RELEASE_DIR)/.stork_address
+
+## STORK_LOG_BASE specifies the basename for heritage Stork log files.
+## Stork uses this macro to create the following output log files:
+## $(STORK_LOG_BASE): Stork server job queue classad collection
+## journal file.
+## $(STORK_LOG_BASE).history: Used to track completed jobs.
+## $(STORK_LOG_BASE).user_log: User level log, also used by DAGMan.
+STORK_LOG_BASE= $(RELEASE_DIR)/log)/storkserver.log
+STORK_LOG = $(RELEASE_DIR)/log/storklog.log
+
+DEBUG = D_FULLDEBUG
+MAX_STORK_LOG = 4000000
+
+## Limits the number of concurrent data placements handled by Stork.
+#MAX_NUM_JOBS = 10
+
+## Limits the number of retries for a failed data placement.
+#MAX_RETRY = 5
+
+## Limits the run time for a data placement job, after which the
+## placement is considered failed.
+#MAXDELAY_INMINUTES = 10
+
+## Directory containing Stork binaries
+BIN_DIR = %(releasedir)s/bin
+
+## Directory containing Stork transfer modules
+MODULE_DIR = %(storkpackage)s/libexec
+
+## Directory to store temporary files and execute transfer modules in
+TMP_DIR = $(RELEASE_DIR)/tmp
+
+## Directory in which server state files should be stored (and the directory in
+## which the Stork server will execute)
+LOG = $(RELEASE_DIR)/log
+
+## Set to true to enable reporting of anonymous usage statistics to the
+## Stork developers. This report contains ONLY the version string and
+## platform Stork was built for.
+## This information helps us improve Stork, so please enable it. :)
+ENABLE_STAT_REPORT = TRUE
+STAT_REPORT_SERVER = stork.cct.lsu.edu
+
+## Additional values passed to transfer modules.
+#RECURSIVE_COPY = FALSE
+#VERIFY_CHECKSUM = FALSE
+#VERIFY_FILESIZE = FALSE
+#NETWORK_CHECK = FALSE
+#CHECKPOINT_TRANSFER = TRUE
+#FILE_CKPT = TRUE
+#SYNC_ONLY = FALSE
+#SET_PERMISSIONS = FALSE
+#TEST_MODE = 0
+#SET_PARALLELISM = 10
+SEC_DEFAULT_AUTHENTICATION_METHODS=ANONYMOUS
+SEC_CLIENT_AUTHENTICATION_METHODS=ANONYMOUS
+SEC_DEFAULT_AUTHENTICATION=NEVER
+SEC_READ_AUTHENTICATION=NEVER
+SEC_WRITE_AUTHENTICATION=NEVER
+SEC_CLIENT_AUTHENTICATION=NEVER
+SEC_DAEMON_AUTHENTICATION=NEVER
\ No newline at end of file

slapos/recipe/stork/template/stork_config.generic

+######################################################################
+##
+##  stork_config
+##
+######################################################################
+#ENABLE_IPV6 = TRUE
+PORT = %(port)s
+NETWORK_INTERFACE = %(ipv4)s
+COLLECTOR_HOST =  %(ipv4)s:$(PORT)
+CONDOR_HOST =  %(ipv4)s
+
+RELEASE_DIR             =  %(releasedir)s
+##Stork startup arguments
+## Start Stork on a well-known port.  Uncomment to simplify connecting to a
+## remote Stork server.  Note: this interface may change in a future release.
+SBIN_DIR = %(releasedir)s/sbin
+STORK= $(SBIN_DIR)/stork_server
+
+ARGS = -p $(STORK_PORT) -f -Serverlog $(STORK_LOG_BASE)
+
+## HOST may specify an optional default remote Stork server and port.
+#HOST  = %(stork_host)s:$(PORT)
+
+HOST  = %(ipv4)s:$(PORT)
+## When Stork starts up, it can place its address (IP and port) into
+## a file. This way, tools running on the local machine don't
+## need an additional "-n host:port" command line option.  This
+## feature can be turned off by commenting out this setting.
+STORK_ADDRESS_FILE = $(RELEASE_DIR)/.stork_address
+
+## STORK_LOG_BASE specifies the basename for heritage Stork log files.
+## Stork uses this macro to create the following output log files:
+## $(STORK_LOG_BASE): Stork server job queue classad collection
+## journal file.
+## $(STORK_LOG_BASE).history: Used to track completed jobs.
+## $(STORK_LOG_BASE).user_log: User level log, also used by DAGMan.
+STORK_LOG_BASE= $(RELEASE_DIR)/log)/storkserver.log
+STORK_LOG = $(RELEASE_DIR)/log/storklog.log
+
+DEBUG = D_FULLDEBUG
+MAX_STORK_LOG = 4000000
+
+## Limits the number of concurrent data placements handled by Stork.
+#MAX_NUM_JOBS = 10
+
+## Limits the number of retries for a failed data placement.
+#MAX_RETRY = 5
+
+## Limits the run time for a data placement job, after which the
+## placement is considered failed.
+#MAXDELAY_INMINUTES = 10
+
+## Directory containing Stork binaries
+BIN_DIR = %(releasedir)s/bin
+
+## Directory containing Stork transfer modules
+MODULE_DIR = %(storkpackage)s/libexec
+
+## Directory to store temporary files and execute transfer modules in
+TMP_DIR = $(RELEASE_DIR)/tmp
+
+## Directory in which server state files should be stored (and the directory in
+## which the Stork server will execute)
+LOG = $(RELEASE_DIR)/log
+
+## Set to true to enable reporting of anonymous usage statistics to the
+## Stork developers. This report contains ONLY the version string and
+## platform Stork was built for.
+## This information helps us improve Stork, so please enable it. :)
+ENABLE_STAT_REPORT = TRUE
+STAT_REPORT_SERVER = stork.cct.lsu.edu
+
+## Additional values passed to transfer modules.
+#RECURSIVE_COPY = FALSE
+#VERIFY_CHECKSUM = FALSE
+#VERIFY_FILESIZE = FALSE
+#NETWORK_CHECK = FALSE
+#CHECKPOINT_TRANSFER = TRUE
+#FILE_CKPT = TRUE
+#SYNC_ONLY = FALSE
+#SET_PERMISSIONS = FALSE
+#TEST_MODE = 0
+#SET_PARALLELISM = 10
\ No newline at end of file

software/stork/instance-stork.cfg

+[buildout]
+
+parts =
+  stork_server
+  stork-submit
+  tunnel-ipv6-stork
+  publish-stork-informations
+
+newest = false
+offline = true
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+# Create all needed directories
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+stork = $${buildout:directory}/log/
+data = $${buildout:directory}/data-jobs/
+bin = $${buildout:directory}/bin/
+sbin = $${buildout:directory}/sbin/
+srv = $${buildout:directory}/srv/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+scripts = $${rootdirectory:etc}/run/
+services = $${rootdirectory:etc}/service/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[stork]
+recipe = slapos.cookbook:stork
+wrapper-path = $${rootdirectory:bin}/stork_raw
+ipv6 = $${slap-network-information:global-ipv6}
+ipv4 =  $${slap-network-information:local-ipv4}
+pid = $${basedirectory:run}/stork.pid
+package = ${stork:location}
+rootdirectory = $${buildout:directory}
+data-dir = $${rootdirectory:data}
+java-bin = ${java:location}/bin
+bin = $${rootdirectory:bin}
+sbin = $${rootdirectory:sbin}
+log= $${rootdirectory:stork}
+tmp= $${rootdirectory:tmp}
+dash = ${dash:location}/bin/dash
+
+environment = 
+  LD_LIBRARY_PATH=${libexpat:location}/lib:${openssl:location}/lib:${zlib:location}/lib
+  PATH=${perl:location}/bin:${globus:location}/bin:${java:location}/bin:${openssl:location}/bin:%(PATH)s
+  HOSTNAME=$${slap-parameter:stork-host}
+
+#stork user parameter
+stork_host = $${slap-parameter:stork-host}
+stork_server = $${slap-parameter:stork_server}
+stork_port = 9621
+machine-role = $${slap-parameter:stork_role}
+x509proxy=$${slap-parameter:x509proxy}
+
+[stork_server]
+recipe = slapos.cookbook:wrapper
+script-file = ${process-svcdeamon:location}/${process-svcdeamon:filename}
+command-line = ${buildout:executable} $${:script-file} $${stork:wrapper-path} $${stork:pid} 10
+wrapper-path = $${basedirectory:services}/stork_server
+
+[stork-submit]
+<= stork
+recipe = slapos.cookbook:stork.submit
+wrapper-path = $${basedirectory:scripts}/stork_submit
+wrapper-check = $${basedirectory:scripts}/stork_check_job
+json_src_url = $${slap-parameter:src_data_url}
+dest_url = $${slap-parameter:dest_data_url}
+data_type=$${slap-parameter:data_type}
+src_from_file = false
+
+# Publish all instance parameters (url of instance)
+[publish-stork-informations]
+recipe = slapos.cookbook:publish
+stork_instance_type = $${stork:machine-role}
+stork_server = $${slap-parameter:stork_server}
+stork_ipv4_address = $${stork:ipv4}
+stork_ipv6_address = $${stork:stork_host}
+data_type=$${slap-parameter:data_type}
+x509proxy=$${slap-parameter:x509proxy}
+
+[tunnel-ipv6-stork]
+recipe = slapos.cookbook:ipv6toipv4
+ipv6 = $${slap-network-information:global-ipv6}
+ipv4 = $${slap-network-information:local-ipv4}
+shell-path = ${dash:location}/bin/dash
+6tunnel-path = ${6tunnel:location}/bin/6tunnel
+ipv6-port = 9621
+ipv4-port = 9621
+runner-path = $${rootdirectory:srv}/6tunnel-stork
+
+[slap-parameter]
+# Default values if not specified
+stork-host = [$${slap-network-information:global-ipv6}]
+# Role server/client Default is server
+stork_server = local
+stork_role = client
+stork-allowed-write = $${:stork-host}
+x509proxy = /tmp/x509up_uXXX
+data_type = transfer
+src_data_url = {"slapos.tar.gz":"http://git.erp5.org/gitweb/slapos.git/snapshot/c4f5ca68af31cd9d8461869d70928c33a03c5aa6.tar.gz"}
+dest_data_url = local

software/stork/instance.cfg

+[buildout]
+
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-stork:output}
\ No newline at end of file

software/stork/scripts/svcdaemon.py

+import os
+import subprocess
+import time
+import signal
+import sys
+
+def get_pid(filename):
+  pid = None
+  if os.path.exists(filename):
+    data = open(pid_file).read()
+    try:
+      pid = int(data)
+    except ValueError:
+      pass
+  return pid
+
+pid_file = None
+def sig_handler(s, frame):
+  print "Killing on signal %s:" % s,
+  global pid_file
+  if pid_file is not None:
+    pid = get_pid(pid_file)
+    if pid is not None:
+      os.kill(pid, signal.SIGTERM)
+      try:
+        os.kill(pid, 0)
+      except Exception:
+        pass
+      else:
+        time.sleep(5)
+        try:
+          os.kill(pid, 0)
+        except Exception:
+          pass
+        else:
+          print 'with SIGKILL...',
+          os.kill(pid, signal.SIGKILL)
+    else:
+      raise ValueError('Pid is none.')
+  print 'done.'
+  sys.exit(0)
+
+signal.signal(signal.SIGINT, sig_handler)
+signal.signal(signal.SIGQUIT, sig_handler)
+signal.signal(signal.SIGTERM, sig_handler)
+
+
+def svcdaemon(args):
+  """Utility script to run daemons in supervisord"""
+  real_binary = args[0]
+  global pid_file
+  pid_file = args[1]
+  subprocess.check_call(real_binary)
+  print 'Started %r' % real_binary
+  while True:
+    time.sleep(5)
+    pid = get_pid(pid_file)
+    if pid is None:
+      raise ValueError('Pid is none')
+    os.kill(pid, 0)
+
+if __name__ == '__main__':
+  args = []
+  args.append(sys.argv[1])
+  args.append(sys.argv[2])
+  if len(sys.argv) > 3:
+    time.sleep(int(sys.argv[3]))
+  svcdaemon(args)
+  exit(0)
\ No newline at end of file

software/stork/software.cfg

+[buildout]
+
+parts =
+  stork
+  slapos-cookbook
+  template
+  template-stork
+  slapos.cookbook-repository
+  check-recipe
+
+extends =
+  ../../component/git/buildout.cfg
+  ../../component/6tunnel/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/stork/buildout.cfg
+  ../../stack/slapos.cfg
+
+# Local development
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = grid-computing
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+mode = 0644
+md5sum = fd88bc6e40bd2f7ecee5ae6d96627420
+
+[template-stork]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-stork.cfg
+output = ${buildout:directory}/template-stork.cfg
+md5sum = 553d89e9e9f69b577b5ef15c4179e0fc
+mode = 0644
+
+[process-svcdeamon]
+recipe = slapos.recipe.download
+mode = 0644
+md5sum = fd09ea9d1c82981f711b78c6dadd3bad
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = ${:_profile_base_location_}/scripts/${:filename}
+filename = svcdaemon.py
+
+
+
+[networkcache]
+# signature certificates of the following uploaders.
+#   Romain Courteaud
+#   Sebastien Robin
+#   Kazuhiko Shiozaki
+#   Cedric de Saint Martin
+#   Yingjie Xu
+#   Gabriel Monnerat
+#   Łukasz Nowak
+#   Test Agent (Automatic update from tests)
+#   Alain Takoudjou
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+  QUUGLQ==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+  BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+  DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+  YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+  XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+  L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+  /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+  LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+  FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+  R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+  hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV
+  BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL
+  BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH
+  vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk
+  3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso
+  67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE
+  RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v
+  PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj
+  z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL
+  KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY
+  vaZhjNYKWQf79l6zXfOvphzJ
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT
+  MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3
+  i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9
+  mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O
+  BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg
+  62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK
+  YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W
+  it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+
+  TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx
+  EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn
+  oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0
+  NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O
+  BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB
+  JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6
+  baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA
+  Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n
+  yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAMNZBmoIOXPBMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMTMyMCAXDTEyMDUwMjEyMDQyNloYDzIxMTIwNDA4MTIwNDI2WjAT
+  MREwDwYDVQQDDAhDT01QLTEzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  6peZQt1sAmMAmSG9BVxxcXm8x15kE9iAplmANYNQ7z2YO57c10jDtlYlwVfi/rct
+  xNUOKQtc8UQtV/fJWP0QT0GITdRz5X/TkWiojiFgkopza9/b1hXs5rltYByUGLhg
+  7JZ9dZGBihzPfn6U8ESAKiJzQP8Hyz/o81FPfuHCftsCAwEAAaNQME4wHQYDVR0O
+  BBYEFNuxsc77Z6/JSKPoyloHNm9zF9yqMB8GA1UdIwQYMBaAFNuxsc77Z6/JSKPo
+  yloHNm9zF9yqMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAl4hBaJy1
+  cgiNV2+Z5oNTrHgmzWvSY4duECOTBxeuIOnhql3vLlaQmo0p8Z4c13kTZq2s3nhd
+  Loe5mIHsjRVKvzB6SvIaFUYq/EzmHnqNdpIGkT/Mj7r/iUs61btTcGUCLsUiUeci
+  Vd0Ozh79JSRpkrdI8R/NRQ2XPHAo+29TT70=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
+  MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  o7aipd6MbnuGDeR1UJUjuMLQUariAyQ2l2ZDS6TfOwjHiPw/mhzkielgk73kqN7A
+  sUREx41eTcYCXzTq3WP3xCLE4LxLg1eIhd4nwNHj8H18xR9aP0AGjo4UFl5BOMa1
+  mwoyBt3VtfGtUmb8whpeJgHhqrPPxLoON+i6fIbXDaUCAwEAAaNQME4wHQYDVR0O
+  BBYEFEfjy3OopT2lOksKmKBNHTJE2hFlMB8GA1UdIwQYMBaAFEfjy3OopT2lOksK
+  mKBNHTJE2hFlMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAaNRx6YN2
+  M/p3R8/xS6zvH1EqJ3FFD7XeAQ52WuQnKSREzuw0dsw12ClxjcHiQEFioyTiTtjs
+  5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
+  x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB8jCCAVugAwIBAgIJANlTfgX/9cX7MA0GCSqGSIb3DQEBBQUAMBExDzANBgNV
+  BAMMBkNPTVAtMDAgFw0xMzA1MzAxMTUyMDhaGA8yMTEzMDUwNjExNTIwOFowETEP
+  MA0GA1UEAwwGQ09NUC0wMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDSGtkh
+  UPOcYMRiL2czmdfeHNB34nXEr5fMpglbA9RO9Cakf6YhK0jU5Y7kzAb6ZlyEzcC+
+  YkYWQZ9WCLwGuUvThg5iYlnBxmksv5Ot+5DKyTL4hdMEPCQ0RaLcXSkbxk56q3Rn
+  EQdYL+IJXS1KGMRWbl4Ql3NhShM9q7vBgyoMuwIDAQABo1AwTjAdBgNVHQ4EFgQU
+  EpO2oyW/sFAS9eojDbYo3hEOOIIwHwYDVR0jBBgwFoAUEpO2oyW/sFAS9eojDbYo
+  3hEOOIIwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQBP02wmekqmeD23
+  90NmwDwyEznDPEJDJybREtP+T++EJ1Q/rcnxIQ5a1iOEWY4RdH87H/+2O/7nDzGs
+  L0KzeG3WDdmnxLMQ5zpG9Qd5twC0GoMl1zf+8f9/t4QE7Dn4IosP3H3dkcRDQRch
+  SKYaJZHMckKx40/hhRsyFDR1Gj215g==
+  -----END CERTIFICATE-----
+  
+[versions]
+pyparsing = 2.0.2
\ No newline at end of file