Update Ajaxplorer, Add resiliency

software/ajaxplorer/instance-apache-export.cfg.jinja2

+# This file is responsible of three things:
+# 1/ Act as "Apache exporter"
+# 2/ Act as "Mariadb backup infrastructure requester"
+# 3/ Act as "Apache" instance
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+extends = {{templateapache}}
+          {{templatepbsreadyexport}}
+
+parts +=
+    {{ parts.replicate("mariadb", "3") }}
+
+# Repeating parts from instance-apache-php.
+# XXX-Cedric: how to simplify this?
+    certificate-authority
+    ca-stunnel
+    logrotate
+    logrotate-entry-apache
+    logrotate-entry-stunnel
+    cron
+    cron-entry-logrotate
+    promise
+  
+    frontend-ajaxupload-promise
+    frontend-website2-promise
+    frontend-website-promise
+#   content-promise
+    configure-ajaxplorer
+    publish-connection-informations
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+# Nothing to do for the exporter. This wrapper is intended
+# to produce the "dump" files that have to be backed up.
+# So, in case of binary DB data, we would back up the ASCII dump
+# files, or a similar data format that can be restored on any machine
+# (i.e. postgres 'custom')
+# In the case of an httpd instance, the src/www directory can be directly
+# pulled from the PBS, we don't need to prepare anything.
+[exporter]
+wrapper = /bin/true
+
+# State that we want to backup srv/www directory, not srv/backup.
+# XXX-Cedric: works well, but doesn't work with big data.
+[rdiff-backup-server]
+path = ${directory:www}
+
+# Add "exporter" parameters to list of published connection parameters
+[publish-connection-informations]
+# XXX-Cedric: Long term goal: could be a recipe that requests an instance and
+# bubbles ALL
+# parameters of the requested instance. Requirement: aggregated publish.
+<= resilient-publish-connection-parameter
+
+# XXX-Cedric: resilient overwrites what's returned from request-mariadb
+# XXX-Cedric: change the request method to return everything from
+#             getConnectionParameterDict()
+[request-mariadb]
+return = ssh-public-key ssh-url notification-id ip url
\ No newline at end of file

software/ajaxplorer/instance-apache-import.cfg.in

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+extends = ${pbsready-import:output}
+
+[directory]
+srv = $${buildout:directory}/srv
+www = $${:srv}/www/
+
+# Nothing to do for the import. Just dummy part that does nothing.
+# For httpd instance, PBS will directly push data to srv/www.
+# XXX-Cedric: write a real backup system.
+[importer]
+wrapper = /bin/true
+
+# State that we want to push to srv/www directory, not srv/backup.
+[rdiff-backup-server]
+path = $${directory:www}
\ No newline at end of file

software/ajaxplorer/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("apache", "3") }}
+  publish-connection-informations
+
+{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }}
+
+# Bubble up the parameters
+[request-apache]
+return = url ssh-public-key ssh-url notification-id ip public_html_backend public_html public_html2_backend public_html2 fileManager_backend fileManager mysql_username mysql_password mysql_database mysql_host mysql_port
+# XXX: hardcoded values
+config = filemanager-domain site-domain site2-domain frontend-software-url number authorized-key notify ip-list namebase mariadb1-computer-guid pbs-mariadb1-computer-guid mariadb2-computer-guid pbs-mariadb2-computer-guid mariadb3-computer-guid pbs-mariadb3-computer-guid
+config-mariadb1-computer-guid = ${slap-parameter:mariadb1-computer-guid}
+config-pbs-mariadb1-computer-guid = ${slap-parameter:pbs-mariadb1-computer-guid}
+config-mariadb2-computer-guid = ${slap-parameter:mariadb2-computer-guid}
+config-pbs-mariadb2-computer-guid = ${slap-parameter:pbs-mariadb2-computer-guid}
+config-mariadb3-computer-guid = ${slap-parameter:mariadb3-computer-guid}
+config-pbs-mariadb3-computer-guid = ${slap-parameter:pbs-mariadb3-computer-guid}
+config-filemanager-domain = ${slap-parameter:filemanager-domain}
+config-site-domain = ${slap-parameter:site-domain}
+config-site2-domain = ${slap-parameter:site2-domain}
+config-frontend-software-url = ${slap-parameter:frontend-software-url}
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+public_html_backend = ${request-apache:connection-public_html_backend}
+public_html = ${request-apache:connection-public_html}
+public_html2_backend = ${request-apache:connection-public_html2_backend}
+public_html2  = ${request-apache:connection-public_html2}
+fileManager_backend = ${request-apache:connection-fileManager_backend}
+fileManager = ${request-apache:connection-fileManager}
+mysql_username = ${request-apache:connection-mysql_username}
+mysql_password = ${request-apache:connection-mysql_password}
+mysql_database = ${request-apache:connection-mysql_database}
+mysql_host = ${request-apache:connection-mysql_host}
+mysql_port = ${request-apache:connection-mysql_port}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of maria should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+mariadb1-computer-guid =
+pbs-mariadb1-computer-guid =
+mariadb2-computer-guid =
+pbs-mariadb2-computer-guid =
+mariadb3-computer-guid =
+pbs-mariadb3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+# Default value if no domain is specified
+site-domain =
+filemanager-domain =
+site2-domain =
+frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg

software/ajaxplorer/instance.cfg

@@ -8,5 +8,15 @@ offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${instance-apache:output}
-mariadb = ${instance-mariadb:output}
\ No newline at end of file
+default = ${instance-apache-php:output}
+resilient = ${instance-resilient:rendered}
+mariadb = ${instance-mariadb:output}
+mariadb-import = ${instance-mariadb-import:output}
+mariadb-export = ${instance-mariadb-export:output}
+apache-import = ${instance-apache-import:output}
+apache-export = ${instance-apache-export:rendered}
+
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
\ No newline at end of file

software/ajaxplorer/software.cfg

@@ -5,61 +5,148 @@ develop =
 
 parts =
   slapos-cookbook
-  subversion
   apache-php
-  instance
-  instance-apache
-  instance-mariadb
+  mariadb
+  imagemagick
+  mydumper
+  mysql-python
+  rdiff-backup
+  dropbear
+  
   template-httpd-conf
   application
+  instance-eggs
   website1-template
   website2-template
+  
+  instance
+  instance-apache-php
+  instance-mariadb
+  
+#Dev with repository
   slapos.cookbook-repository
   check-recipe
+  
+#Contains the importer and exporter recipes for mariadb
+  instance-mariadb-import
+  instance-mariadb-export
+
+#Contains the importer and exporter recipes for apache
+  instance-apache-import
+  instance-apache-export
 
 extends =
-  ../../stack/slapos.cfg
   ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dropbear/buildout.cfg
   ../../component/logrotate/buildout.cfg
-  ../../component/subversion/buildout.cfg
   ../../component/gzip/buildout.cfg
+  ../../component/glib/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/stunnel/buildout.cfg
+  ../../component/mariadb/buildout.cfg
+  ../../component/mydumper/buildout.cfg
+  ../../component/mysql-python/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/imagemagick/buildout.cfg
+  ../../component/perl/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/sqlite3/buildout.cfg
+  ../../component/zlib/buildout.cfg
   ../../component/git/buildout.cfg
+  ../../stack/resilient/buildout.cfg
+  ../../stack/slapos.cfg
+
+[instance-eggs]
+recipe = zc.recipe.egg
+eggs =
+    plone.recipe.command
+    ${lxml-python:egg}
 
 [instance]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 4c7936accb3658871b635158198b7905
+md5sum = f9ee461a59e83592c19581dc9efc7dd9
 
-[instance-apache]
+[instance-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apachephp.cfg
+url = ${:_profile_base_location_}/instance-apachephp.cfg.in
 output = ${buildout:directory}/template-apachephp.cfg
-md5sum = ef5c61127a21c016014021477d1791e3
+md5sum = 0c574497ad00ef2e1ce684ceccda3f5e
+mode = 0644
+
+[instance-apache-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-apache-import.cfg.in
+output = ${buildout:directory}/instance-apache-import.cfg
+#md5sum = f1dc2a71d362b5d2d36481ffefdd2293
+mode = 0644
+
+[instance-apache-export]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-apache-export.cfg.jinja2
+rendered = ${buildout:directory}/instance-apache-export.cfg
+
+context = key templateapache instance-apache-php:output
+          key templatepbsreadyexport pbsready-export:output
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+#md5sum = 24c5d0db063e5343a3cb0db9d5fbc738
+mode = 0644
+
+[instance-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
+rendered = ${buildout:directory}/instance-resilient.cfg
+test = ${template-parts:destination}
+
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 7b7b06d35b220e466da576ffa380bc93
 mode = 0644
 
 [instance-mariadb]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/../../stack/lamp/mariadb/instance-mariadb.cfg.in
 output = ${buildout:directory}/template-mariadb.cfg
+#md5sum
+mode = 0644
+
+[instance-mariadb-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../../stack/lamp/mariadb/instance-mariadb-import.cfg.in
+output = ${buildout:directory}/instance-mariadb-import.cfg
+#md5sum = faf5826f4b27e362f34ad17db54b8c9a
+mode = 0644
+
+[instance-mariadb-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../../stack/lamp/mariadb/instance-mariadb-export.cfg.in
+output = ${buildout:directory}/instance-mariadb-export.cfg
+#md5sum = 0352896921ca1766b64f0b54f0f8c27e
 mode = 0644
 
 [application]
 recipe = hexagonit.recipe.download
-url = http://garr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/dev-channel/4.3.4/ajaxplorer-core-4.3.4.tar.gz
-md5sum = 2f2ff8bda7bbe841ef0e870c724eb74f
+url = http://freefr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/stable-channel/5.0.1/ajaxplorer-core-5.0.1.tar.gz
+md5sum = 805cd6a473b3d3516388450afc5a0e6a
 strip-top-level-dir = true
 
 [website]
 recipe = z3c.recipe.mkdir
-path1 = ${application:location}/data/files/website
-path2 = ${application:location}/data/files/website2
+path1 = ${application:location}/data/files/public_html
+path2 = ${application:location}/data/files/public_html2
 paths =
   ${:path1}
   ${:path2}
@@ -68,22 +155,26 @@ paths =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/index.html.in
 output = ${website:path1}/index.html
-md5sum = c5695762361b801c284ee23a150cd1f1
+md5sum = 979c36dc7dbf9d1e89818060f32a39b7
 mode = 0644
 
+folder = public_html
+
 [website2-template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/index.html.in
 output = ${website:path2}/index.html
-md5sum = c5695762361b801c284ee23a150cd1f1
+md5sum = 979c36dc7dbf9d1e89818060f32a39b7
 mode = 0644
 
+folder = public_html2
+
 [template-httpd-conf]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/templates/${:filename}
 mode = 0644
 filename = apache.conf.in
-md5sum = 6788381fadd8ca96f892d33df3163da0
+md5sum = ffb1105e2620e9e9f1f45093b67a3df8
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 # Local development
@@ -216,51 +307,46 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
-#Error: Couldn't install: lxml 3.1.2
-lxml = 3.1.1
-# Numpy 1.7.0 doesn't install well
-numpy = 1.6.2
-# websockify 0.4.1 doesn't install well
-websockify = 0.3.0
-
-setuptools = 0.6c12dev-r88846
-
-Jinja2 = 2.7
-MarkupSafe = 0.18
-Werkzeug = 0.8.3
+Jinja2 = 2.6
+Werkzeug = 0.9.1
 buildout-versions = 1.7
+cliff = 1.4
+cmd2 = 0.6.5.1
 inotifyx = 0.2.0-1
+itsdangerous = 0.22
+lxml = 3.2.1
 meld3 = 0.6.10
 netaddr = 0.7.10
+netifaces = 0.8-1
 plone.recipe.command = 1.1
+prettytable = 0.7.2
 pytz = 2013b
+rdiff-backup = 1.0.5
+requests = 1.2.3
 slapos.core = 0.35.1
 slapos.recipe.build = 0.11.6
 slapos.recipe.cmmi = 0.1.1
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
+supervisor = 3.0b2
 xml-marshaller = 0.9.7
-z3c.recipe.mkdir = 0.5
+z3c.recipe.mkdir = 0.6
 
 # Required by:
 # slapos.core==0.35.1
-Flask = 0.9
+Flask = 0.10.1
 
 # Required by:
-# slapos.cookbook==0.78.1.dev
+# slapos.cookbook==0.78.2.dev
 lock-file = 2.0
 
 # Required by:
 # slapos.core==0.35.1
-netifaces = 0.8-1
-
-# Required by:
-# slapos.core==0.35.1
-pyflakes = 0.7.2
+pyflakes = 0.7.3
 
 # Required by:
-# slapos.core==0.35.1
-supervisor = 3.0b1
+# plone.recipe.command==1.1
+setuptools = 0.8
 
 # Required by:
 # slapos.core==0.35.1
@@ -268,4 +354,4 @@ unittest2 = 0.5.1
 
 # Required by:
 # slapos.core==0.35.1
-zope.interface = 4.0.5
\ No newline at end of file
+zope.interface = 4.0.5

software/ajaxplorer/templates/apache.conf.in

@@ -3,9 +3,9 @@
 
 # Basic server configuration
 PidFile "${:pid_file}"
-Listen ${:ip}:8070
-Listen ${:ip}:8080
-Listen ${:ip}:8090
+Listen ${:ip}:${:port}
+Listen ${:ip}:${:port2}
+Listen ${:ip}:${:port3}
 PHPINIDir ${:php_ini_dir}
 ServerAdmin someone@email
 TypesConfig conf/mime.types
@@ -21,10 +21,10 @@ LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Ag
 LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b" common
 CustomLog "${:access_log}" common
 
-NameVirtualHost ${:ip}:8090
-NameVirtualHost ${:ip}:8080
-NameVirtualHost ${:ip}:8070
-<VirtualHost ${:ip}:8070>
+NameVirtualHost ${:ip}:${:port2}
+NameVirtualHost ${:ip}:${:port}
+NameVirtualHost ${:ip}:${:port3}
+<VirtualHost ${:ip}:${:port2}>
   #ServerName www.example.com
   
   # Directory protection
@@ -45,7 +45,7 @@ NameVirtualHost ${:ip}:8070
 
 </VirtualHost>
 
-<VirtualHost ${:ip}:8080>
+<VirtualHost ${:ip}:${:port}>
   #ServerName www.example.com
   
   # Directory protection
@@ -55,18 +55,18 @@ NameVirtualHost ${:ip}:8070
       Require all denied
   </Directory>
   
-  <Directory ${:document_root}data/files/website/>
+  <Directory ${:public_html}>
     Options FollowSymLinks
     AllowOverride All
     Require all granted
   </Directory>
   
-  DocumentRoot ${:document_root}data/files/website/
+  DocumentRoot ${:public_html}
   DirectoryIndex index.html index.php
 
 </VirtualHost>
 
-<VirtualHost ${:ip}:8090>
+<VirtualHost ${:ip}:${:port3}>
   #ServerName www.example.com
   
   # Directory protection
@@ -76,17 +76,25 @@ NameVirtualHost ${:ip}:8070
       Require all denied
   </Directory>
   
-  <Directory ${:document_root}data/files/website2/>
+  <Directory ${:public_html2}>
     Options FollowSymLinks
     AllowOverride All
     Require all granted
   </Directory>
   
-  DocumentRoot ${:document_root}data/files/website2/
+  DocumentRoot ${:public_html2}
   DirectoryIndex index.html index.php
 
 </VirtualHost>
 
+#Update PHP.INI
+
+php_value include_path ".:${:php_lib_path}"
+php_value upload_max_filesize 20M
+php_value post_max_size 20M
+php_value output_buffering none
+
+
 # List of modules
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so

software/ajaxplorer/templates/index.html.in

 <html>
 <head>
-<title>New Document - Created By AjaXplorer</title>
+<title>Empty Website</title>
 <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
 </head>
 <body bgcolor="#FFFFFF" text="#000000">
-  <h2>Hi, this website folder is still empty. Please go to your File Manager administration and replace this folder by your new website files.</h2>
+  <h3>Hi, this website folder is still empty. Please go to your File Manager administration (filemanager-url) and 
+  upload your files into ${:folder} folder.</h3>
 </body>
 </html>