Now use Stork app to download file for BOINC apps

component/condor/buildout.cfg

@@ -35,8 +35,8 @@ depends =
   ${dash:location}
   ${zlib:location}
 
-x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb
-x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68
+x86 = https://storage.host.cloud.univ-paris13.fr/data/public/00198e.php?dl=1 957a291290d4da089d004f52adc4a490
+x86-64 = https://storage.host.cloud.univ-paris13.fr/data/public/c1a6ff.php?dl=1 41c697e6c88f750b0e5c98baf0d260a5
 
 script =
   if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')

slapos/recipe/boinc/__init__.py

@@ -26,11 +26,8 @@
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
 import os
-import subprocess
 import pwd
 import json
-import signal
-import zc.buildout
 from urlparse import urlparse
 
 class Recipe(GenericBaseRecipe):
@@ -232,21 +229,19 @@ class App(GenericBaseRecipe):
   """This recipe allow to deploy an scientific applications using boinc
   Note that recipe use depend on boinc-server parameter"""
 
-  def downloadFiles(self, app):
+  def getDownloadFiles(self, app, files_dict, prefix):
     """This is used to download app files if necessary and update options values"""
+    
     for key in ('input-file', 'template-result', 'template-wu', 'binary'):
-      param = app[key]
-      if param and (param.startswith('http') or param.startswith('ftp')):
-        #download the specified file
-        cache = os.path.join(self.options['home'].strip(), 'tmp')
-        downloader = zc.buildout.download.Download(self.buildout['buildout'],
-                        hash_name=True, cache=cache)
-        path, _ = downloader(param, md5sum=None)
-        mode = 0600
-        if key == 'binary':
-          mode = 0700
-        os.chmod(path, mode)
-        app[key] = path
+      src = app[key]
+      if not src:
+        continue
+      if src.startswith('/'):
+        src = "file:" + src
+      filename = (prefix + '_' + key).replace('-', '_')
+      files_dict[filename] = src
+      app[key] = os.path.join(self.options['download-folder'].strip(),
+                              filename)
 
   def getAppList(self):
     """Load parameters,
@@ -262,7 +257,7 @@ class App(GenericBaseRecipe):
       for version in app_list[app]:
         current_app = app_list[app][version]
         #Use default value if empty and Use_default is True
-        #Initialize all values to empty if not define by the user
+        #Initialize all values to empty if not defined by the user
         if current_app['use_default']:
           current_app['template-result'] = current_app.get('template-result',
                         default_template_result).strip()
@@ -300,6 +295,7 @@ class App(GenericBaseRecipe):
   def install(self):
 
     app_list = self.getAppList()
+    download_list_dict = dict()
 
     path_list = []
     package = self.options['boinc'].strip()
@@ -343,7 +339,9 @@ class App(GenericBaseRecipe):
       for version in app_list[appname]:
         if not app_list[appname][version]:
           continue
-        self.downloadFiles(app_list[appname][version])
+        # Get the list of files to download
+        self.getDownloadFiles(app_list[appname][version],
+                          download_list_dict, appname+'_'+version)
         platform = app_list[appname][version]['platform']
         application = os.path.join(apps_dir, appname, version, platform)
         if app_list[appname][version]['binary'] and not platform:
@@ -369,6 +367,9 @@ class App(GenericBaseRecipe):
           '%s.configure.deployApp' % __name__, parameter
         )
         path_list.append(deploy_app)
+    
+    with open(self.options['download-json-file'].strip(), 'w') as file_list:
+      file_list.write(json.dumps(download_list_dict))
 
     return path_list
 

slapos/recipe/boinc/configure.py

@@ -191,6 +191,11 @@ def services(args):
 
 def deployApp(args):
   """Deploy Boinc App with lock"""
+  print "Check if all files have been downloaded..."
+  for name in ['t_input', 'binary', 't_result', 't_wu']:
+    if args[name]:
+      checkFile(args[name], 5)
+  
   print "Asking to enter in execution with lock mode..."
   with LockFile(args['lockfile'], wait=True):
     print "acquire the lock file..."

software/boinc/input/input_file

-test string
+Et est admodum mirum videre plebem innumeram mentibus ardore quodam infuso cum dimicationum curulium eventu pendentem. haec similiaque memorabile nihil vel serium agi Romae permittunt. ergo redeundum ad textum.
\ No newline at end of file
+
+Oportunum est, ut arbitror, explanare nunc causam, quae ad exitium praecipitem Aginatium inpulit iam inde a priscis maioribus nobilem, ut locuta est pertinacior fama. nec enim super hoc ulla documentorum rata est fides.
+
+Iam virtutem ex consuetudine vitae sermonisque nostri interpretemur nec eam, ut quidam docti, verborum magnificentia metiamur virosque bonos eos, qui habentur, numeremus, Paulos, Catones, Galos, Scipiones, Philos; his communis vita contenta est; eos autem omittamus, qui omnino nusquam reperiuntur.
+
+Metuentes igitur idem latrones Lycaoniam magna parte campestrem cum se inpares nostris fore congressione stataria documentis frequentibus scirent, tramitibus deviis petivere Pamphyliam diu quidem intactam sed timore populationum et caedium, milite per omnia diffuso propinqua, magnis undique praesidiis conmunitam.
+
+Cum saepe multa, tum memini domi in hemicyclio sedentem, ut solebat, cum et ego essem una et pauci admodum familiares, in eum sermonem illum incidere qui tum forte multis erat in ore. Meministi enim profecto, Attice, et eo magis, quod P. Sulpicio utebare multum, cum is tribunus plebis capitali odio a Q. Pompeio, qui tum erat consul, dissideret, quocum coniunctissime et amantissime vixerat, quanta esset hominum vel admiratio vel querella.
\ No newline at end of file

software/boinc/software.cfg

@@ -62,7 +62,7 @@ md5sum = 66d7ec85ce15e65d2858c11b75fb9574
 url = ${:_profile_base_location_}/input/input_file
 filename = input_file
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-md5sum = 6f8db599de986fab7a21625b7916589c
+md5sum = ad3ca0f59b475ada29dbf68f2951e2a7
 
 [boinc-application]
 app-list = {"${application:app-name}":{"${application:version}":{"use_default":true, "wu-number":2, "input-file":"${template_input:location}/${template_input:filename}", "binary":"${application:location}/${application:filename}"}}}
@@ -73,6 +73,7 @@ template-wu = ${template_wu:location}/${template_wu:filename}
 platform = x86_64-pc-linux-gnu
 extension = 
 
+
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git

stack/boinc/buildout.cfg

@@ -7,6 +7,7 @@ parts =
   template-mariadb
   template-boinc
   template-httpd-conf
+  stork
 
 extends =
   ../../component/boinc/buildout.cfg
@@ -20,6 +21,8 @@ extends =
   ../../component/lxml-python/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../component/6tunnel/buildout.cfg
+  ../../component/stork/buildout.cfg
   ../slapos.cfg
   
 [instance-egg]
@@ -34,7 +37,7 @@ eggs =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-boinc.cfg
 output = ${buildout:directory}/template-boinc.cfg
-md5sum = 986d45af5734c917b49e489bd8c10f3d
+md5sum = bde689de1435bcfc1431692e487ae7ba
 mode = 0644
 
 #Template for deploying MySQL Database Server
@@ -45,6 +48,9 @@ output = ${buildout:directory}/template-mariadb.cfg
 #md5sum = 8142244a3087a23e002a08225579528a  
 mode = 0644
 
+[mariadb]
+keep-compile-dir = false
+
 [template-httpd-conf]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
@@ -62,6 +68,24 @@ platform = x86_64-pc-linux-gnu
 [boinc-application]
 app-list = {}
 
+######################################################################
+####                  STORK INTEGRATION PART                      ####
+
+[template-stork]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../../software/stork/instance-stork.cfg
+output = ${buildout:directory}/template-stork.cfg
+#md5sum = 42985a425369e4f93211cfaad21d8eb8
+mode = 0644
+
+[process-svcdeamon]
+<= template-base
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+url = ${:_profile_base_location_}/../../software/stork/scripts/${:filename}
+filename = svcdaemon.py
+
+#####################END STORK###########################################
+
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud

stack/boinc/instance-boinc.cfg

 [buildout]
 
 parts =
+  stork_server
+  submit-download
   boinc-server
   boinc-app
   certificate-authority
@@ -19,10 +21,11 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-#temporary solution
 newest = false
-offline = false
+offline = true
 
+extends = 
+  ${template-stork:output}
 
 # Creation of all needed directories
 [rootdirectory]
@@ -33,6 +36,7 @@ srv = $${buildout:directory}/srv/
 bin = $${buildout:directory}/bin/
 tmp = $${buildout:directory}/tmp/
 project = $${buildout:directory}/projects/
+download-dir = $${buildout:directory}/files/
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -129,7 +133,7 @@ pid_file = $${basedirectory:run}/apache.pid
 lock_file = $${basedirectory:run}/apache.lock
 cgid_sock = $${basedirectory:run}/cgid.sock
 ip = $${slap-network-information:global-ipv6}
-port = 8080
+port = 8081
 error_log = $${directory:httpd-log}/error.log
 access_log = $${directory:httpd-log}/access.log
 php_ini_dir = $${directory:php-ini-dir}
@@ -173,6 +177,16 @@ environment-section =
 template = 
 configuration = 
 
+[submit-download]
+<= stork
+recipe = slapos.cookbook:stork.submit
+wrapper-check = $${basedirectory:scripts}/stork_check_job
+wrapper-path = $${basedirectory:scripts}/submit_download_files
+json_src_url = $${boinc-app:download-json-file}
+dest_url = file:$${rootdirectory:download-dir}
+src_from_file = true
+data_type= transfer
+
 [boinc-passwd]
 recipe = slapos.cookbook:generate.password
 storage-path = $${rootdirectory:etc}/.boincpasswd
@@ -222,6 +236,8 @@ mysql-port = $${stunnel:local-port}
 [boinc-app]
 <= boinc-server
 recipe = slapos.cookbook:boinc.app
+download-folder = $${rootdirectory:download-dir}
+download-json-file = $${rootdirectory:download-dir}/download.json
 #appname and version is require to update any existing application
 #otherwise, the recipe would try to install a new one
 boinc-app-list  = $${slap-parameter:boinc-app-list}
@@ -353,7 +369,7 @@ curl_path = ${curl:location}/bin/curl
 
 [slap-parameter]
 #Default config value for boinc
-force-ipv6 = False
+force-ipv6 = True
 project = boinc_test
 full-name = Boinc Project SAMPLE
 copyright-holder = REPLACE WITH COPYRIGHT HOLDER