Merge master

component/cloudooo/buildout.cfg

@@ -22,7 +22,7 @@ git-executable = ${git:location}/bin/git
 
 [cloudooo]
 recipe = zc.recipe.egg
-python = python2.6
+python = python2.7
 extra-paths = ${cloudooo-repository:location}
 eggs =
   ${lxml-python:egg}

component/python-2.7/buildout.cfg

@@ -44,6 +44,9 @@ configure-options =
   --enable-unicode=ucs4
   --with-system-expat
   --with-threads
+# Profiled build:
+make-binary =
+make-targets = make profile-opt && make install
 
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes

slapos/recipe/generic_mysql/mysql.py

@@ -68,6 +68,7 @@ def updateMysql(args):
   while True:
     mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--user=root']
     if 'socket' in conf:
+      mysql_upgrade_list.insert(1)
       mysql_upgrade_list.append('--socket=' + conf['socket'])
     mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     result = mysql_upgrade.communicate()[0]
@@ -82,6 +83,7 @@ def updateMysql(args):
         print "No need to upgrade MySQL database"
       mysql_list = [conf['mysql_binary'].strip(), '-B', '--user=root']
       if 'socket' in conf:
+        mysql_list.insert(1)
         mysql_list.append('--socket=' + conf['socket'])
       mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
           stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

slapos/recipe/haproxy/__init__.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
+from zc.buildout import UserError
 
 class Recipe(GenericBaseRecipe):
   """
@@ -80,6 +81,10 @@ class Recipe(GenericBaseRecipe):
       backend_dict = {
         self.options['name']: (self.options['port'], backend_list),
       }
+    if not backend_dict:
+      # Fail installation if backend_dict is empty, otherwise partition may be
+      # advertised as available although it provides no service.
+      raise UserError('backend-dict is empty')
 
     server_snippet_filename = self.getTemplateFilename(
       'haproxy-server-snippet.cfg.in')

software/cloudooo/software.cfg

@@ -14,8 +14,6 @@ parts +=
 # Local development
   slapos.cookbook-repository
   check-recipe
-  slapos.cookbook-python2.6
-  slapos.recipe.template-python2.6
 # Create instance template
   template
 
@@ -38,19 +36,6 @@ stop-on-error = true
 update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
-[slapos.cookbook-python2.6]
-recipe = zc.recipe.egg
-eggs = slapos.cookbook
-scripts =
-python = python2.6
-ugly-depend-on = ${slapos.cookbook-repository:command} ${slapos.cookbook-repository:update-command}
-
-[slapos.recipe.template-python2.6]
-recipe = zc.recipe.egg
-eggs = slapos.recipe.template
-scripts =
-python = python2.6
-
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}.in

software/erp5-util-testing/instance.cfg

@@ -28,6 +28,8 @@ repository = ${erp5-util-repository:location}
 recipe = slapos.cookbook:egg_test
 run-test-suite = $${create-directory:bin}/runTestSuite
 run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
+run-scalability-test-suite = $${create-directory:bin}/runScalabilityTestSuite
+run-scalability-test-suite-binary = $${buildout:bin-directory}/runScalabilityTestSuite
 # The list of executables should be defined here and a combination
 # of tests should dynamically generated.
 #python-list = $${}

stack/cloudooo.cfg

@@ -36,7 +36,6 @@ extends =
   ../component/libreoffice-bin/buildout.cfg 
   ../component/libpng/buildout.cfg
   ../component/lxml-python/buildout.cfg
-  ../component/python-2.6/buildout.cfg
   ../component/python-2.7/buildout.cfg
   ../component/xorg/buildout.cfg
   ../component/fonts/buildout.cfg
@@ -77,7 +76,6 @@ parts =
   poppler
   ffmpeg
 
-  bootstrap2.6
   rdiff-backup
   cloudooo
 
@@ -85,9 +83,6 @@ parts =
 develop +=
   ${:parts-directory}/cloudooo
 
-[bootstrap2.6]
-python = python2.6
-
 [versions]
 # Use SlapOS patched zc.buildout
 zc.buildout = 1.6.0-dev-SlapOS-006

stack/erp5/buildout.cfg

@@ -3,6 +3,8 @@
 develop =
   ${:parts-directory}/slapos.cookbook-repository
   ${:parts-directory}/cloudooo-repository
+# not really just erp5-util but all erp5
+  ${:parts-directory}/erp5.dev-repository
 
 extensions =
   slapos.zcbworkarounds
@@ -14,6 +16,8 @@ find-links =
     http://dist.repoze.org
     http://www.nexedi.org/static/packages/source/
     http://www.owlfish.com/software/wsgiutils/download.html
+    https://pypi.python.org/packages/source/z/z3c.etestbrowser/
+    https://pypi.python.org/packages/source/z/zope.testbrowser/
 
 allow-hosts += pybrary.net
 
@@ -102,6 +106,8 @@ parts =
   eggs
   testrunner
   test_suite_runner
+  scalability_test_suite_runner
+  performance_tester
 
 # basic Xorg
   libXdmcp
@@ -131,12 +137,38 @@ parts =
 
 # Local development
   slapos.cookbook-repository
-  check-recipe
+  erp5.dev-repository
+
+  erp5.util
   slapos.cookbook
+
   slapos.recipe.template
 # Create instance template
   template
 
+  check-recipe
+  erp5.util-check
+
+
+[erp5.dev-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://192.168.242.75:1235/erp5.git
+branch = scalability-master2
+git-executable = ${git:location}/bin/git
+
+[erp5.util-check]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+
+[erp5.util]
+recipe = zc.recipe.egg
+eggs = erp5.util
+scripts =
+ugly-depend-on = ${erp5.dev-repository:repository} ${erp5.dev-repository:branch}
+
+
 [download-base]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:filename}
@@ -423,6 +455,43 @@ initialization =
   repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
   sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
 
+[scalability_test_suite_runner]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+extra-paths = ${eggs:extra-paths}
+entry-points =
+  runScalabilityTestSuite=erp5.util.scalability.runScalabilityTestSuite:main
+scripts = runScalabilityTestSuite
+initialization =
+  import os
+  import sys
+  import erp5.util
+  [Products.__path__.insert(0, p) for p in reversed(os.environ.get('INSERT_PRODUCTS_PATH', '').split(':')) if p]
+  import Zope2
+  os.environ['SOFTWARE_HOME'] = os.path.abspath(os.path.dirname(os.path.dirname(Zope2.__file__)))
+  os.environ['ZOPE_SCRIPTS'] = ''
+  repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
+  sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
+  
+[performance_tester]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+extra-paths = ${eggs:extra-paths}
+entry-points =
+  performance_tester=erp5.util.benchmark.performance_tester:main
+scripts = performance_tester
+initialization =
+  import os
+  import sys
+  import erp5.util
+  [Products.__path__.insert(0, p) for p in reversed(os.environ.get('INSERT_PRODUCTS_PATH', '').split(':')) if p]
+  import Zope2
+  os.environ['SOFTWARE_HOME'] = os.path.abspath(os.path.dirname(os.path.dirname(Zope2.__file__)))
+  os.environ['ZOPE_SCRIPTS'] = ''
+  repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
+  sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
+
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -461,7 +530,7 @@ eggs =
   feedparser
   argparse
   validictory
-  erp5.util
+#  erp5.util
   huBarcode
   qrcode
   spyne
@@ -505,6 +574,15 @@ eggs =
 # Needed for parsing .po files from our Localizer subset
   polib
 
+
+
+# performance_tester dependencies
+  z3c.etestbrowser
+  zope.testbrowser
+# runScalabilityTestSuite dependencie
+  slapos.core
+  psutil
+
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance
@@ -518,11 +596,12 @@ scripts =
   web_checker_utility = erp5.util.webchecker:web_checker_utility
 
 extra-paths =
-  ${erp5:location}
+#  ${erp5:location}
+  ${erp5.dev-repository:location}
 
 [zodbanalyze]
 recipe = zc.recipe.egg
-python = python2.6
+python = python2.7
 eggs =
   ZODB3
   erp5.util
@@ -547,11 +626,19 @@ scripts =
   zodbpack
 
 [cloudooo]
+
+
 # cloudooo section tries to use python2.6, which doesn't exist anymore.
 # Make it use 2.7 instead.
 python = python2.7
 
 [versions]
+
+z3c.etestbrowser = 2.0.0
+zope.testbrowser =
+slapos.core = 0.32.3
+psutil = 
+
 # pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
 Acquisition = 2.13.7nxd001
 Products.DCWorkflow = 2.2.3nxd002
@@ -600,6 +687,16 @@ Products.CMFDefault = 2.2.2
 Products.CMFTopic = 2.2.1
 Products.CMFUid = 2.2.1
 
+# newer version requires zope.traversing>=4.0.0a2.
+zope.app.appsetup = 3.16.0
+
+# newer version requires zope.i18n>=4.0.0a3
+zope.app.publication = 3.14.0
+
+# BBB: Temporarily keep zope.app.testing awaiting we use newer version of CMF
+#      (for tests like testCookieCrumbler).
+zope.app.testing = 3.8.1
+
 # Pinned versions
 Flask = 0.9
 GitPython = 0.3.2.RC1
@@ -617,6 +714,7 @@ Products.LongRequestLogger = 1.1.0
 Products.MimetypesRegistry = 2.0.4
 Products.PluginRegistry = 1.3
 Products.TIDStorage = 5.4.8
+Products.ZSQLMethods = 2.13.4
 Pygments = 1.6
 StructuredText = 2.11.1
 WSGIUtils = 0.7
@@ -631,10 +729,12 @@ coverage = 3.6
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
 erp5.recipe.cmmiforcei686 = 0.1.3
-erp5.util = 0.4.34
+#erp5.util = 0.4.34
+erp5.util = 
 erp5diff = 0.8.1.5
 eventlet = 0.12.1
 feedparser = 5.1.3
+five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
 fpconst = 0.7.2
 gitdb = 0.5.4

stack/erp5/instance-scalability.cfg.in

@@ -7,6 +7,7 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+# Request erp5-cluster instance
 [erp5-cluster]
 recipe = slapos.cookbook:request.serialised
 sla = computer_guid
@@ -25,24 +26,26 @@ config =
 extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
-[launcher-param]
-binary-path = ${rootdirectory:bin}/launcher
-wrapper-path = ${rootdirectory:bin}/launcher
-log-path = ${basedirectory:log}/launcher.log
-test-suite-title = {{ slapparameter_dict.get('test-suite-title', '') }}
 
+# Create wrapper and execute command with parameters
 [binary-wrap-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = "${launcher-param:binary-path} --test_suite_title ${launcher-param:test-suite-title} --log ${launcher-param:log-path}"
-output = ${launcher-param:binary-path}
+log-path = ${basedirectory:log}/runScalabilityTestSuite.log
+binary-path = {{ bin_directory }}/runScalabilityTestSuite
+wrapper-path = ${rootdirectory:bin}/runScalabilityTestSuite
+output = ${binary-wrap-launcher:binary-path}
+# TODO : get the erp5-cluster main url (apache listening user port ?)
+erp5-url = http://foo.bar
+# Run scalability test suite wrapper command with parameters
+# -\ dirty way to conserve spaces
+command-line = "${binary-wrap-launcher:binary-path}" --erp5-url ${binary-wrap-launcher:erp5-url} --test-result-path {{ slapparameter_dict.get('test-result-path') }} --revision {{ slapparameter_dict.get('revision') }} --node-title {{ slapparameter_dict.get('node-title') }} --test-suite-master-url {{ slapparameter_dict.get('test-suite-master-url') }} --log-path ${binary-wrap-launcher:log-path}
 return = url
 
 
+# Create partition's directories
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
 log = ${rootdirectory:var}/log
-run = ${rootdirectory:var}/run
-services = ${rootdirectory:etc}/run
 
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory

stack/erp5/instance.cfg.in

@@ -240,6 +240,18 @@ extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type cluster-zope
 
+[dynamic-template-scalability]
+< = jinja2-template-base
+template = {{ template_scalability }}
+filename = instance-scalability.cfg
+extensions = jinja2.ext.do
+bin_directory = {{ bin_directory }}
+extra-context =
+    key bin_directory dynamic-template-scalability:bin_directory
+    key local_bt5_repository dynamic-template-cluster-erp5-parameters:local-bt5-repository
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type scalability
+
 [dynamic-template-kumofs-parameters]
 dash-location = {{ dash_location }}
 dcron-location = {{ dcron_location }}