Introduce new stack for sync or/and clone instances

This stack allow to sync two instance. One instance will have server scripts
the oter one will have client script. It uses certificate authority stack and sshd stack
to work. The server can describe a list of files that the client can sync, then he also
approuve if the client can sync or not by adding the client ssh-key to his authorized_keys.

stack/sync/buildout.cfg

+[buildout]
+
+extends =
+  ../../component/rsync/buildout.cfg
+  ../../component/openssh/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/bash/buildout.cfg
+  ../../stack/logrotate/buildout.cfg
+
+parts = 
+  template-instance-sync
+  template-server-sync
+
+[slapos-cookbook]
+interpreter = python.eggs
+eggs += 
+  slapos.toolbox
+
+[template-instance-sync]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-sync.cfg.jinja2.in
+rendered = ${buildout:directory}/template-sync.cfg
+md5sum = 3158b940beb73226f5c974ab682f8f29
+context =
+  key template_logrotate_base template-logrotate-base:rendered
+  raw certificate_request_bin ${buildout:directory}/bin/ca-web-request
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw bash_executable_location ${bash:location}/bin/bash
+  raw dcron_executable_location ${dcron:location}/sbin/crond
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  raw rsync_executable_location ${rsync:location}/bin/rsync
+  raw openssh_location ${openssh:location}
+  raw python_executable ${buildout:executable}
+  raw template_python_sync ${template-python-sync:location}/${template-python-sync:filename}
+  raw template_sync_sh ${template-sync.sh:location}/${template-sync.sh:filename}
+
+[template-server-sync]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-sync-server.cfg.jinja2.in
+rendered = ${buildout:directory}/template-server-sync.cfg
+md5sum = 911fe00850ddb3a16a1593ed0b9b6781
+context =
+  key template_logrotate_base template-logrotate-base:rendered
+  raw bash_executable_location ${bash:location}/bin/bash
+  raw rsync_executable_location ${rsync:location}/bin/rsync
+  raw template_server_sync_sh ${prepare-sync-server.sh:location}/${prepare-sync-server.sh:filename}
+  raw template_sync_cfg ${template-sync.cfg:location}/${template-sync.cfg:filename}
+
+[template-sync-download-base]
+recipe = hexagonit.recipe.download
+ignore-existing = true
+download-only = true
+url = ${:_profile_base_location_}/template/${:filename}
+mode = 0644
+
+[template-instance-clone]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/template-instance-clone.cfg.jinja2
+md5sum = 1ef4990a130de8331630f80641bb7213
+filename = template-instance-clone.cfg.jinja2.in
+
+[template-sync.sh]
+<= template-sync-download-base
+md5sum = 14db8beae2d1a027e8339d0879f32a90
+filename = template-sync.sh.in
+
+[prepare-sync-server.sh]
+<= template-sync-download-base
+md5sum = 68fc4916a77fe4e48d6257062047c77e
+filename = prepare-sync-server.sh.in
+
+[template-python-sync]
+<= template-sync-download-base
+md5sum = 2798483a9bf2e6961324c8d95504b238
+filename = python_rsync.py
+mode = 0755
+
+[template-python-bang]
+<= template-sync-download-base
+md5sum = 55d34fa7afd43f1b0815a2667be629c2
+filename = instance_bang.py
+mode = 0755
+
+[template-sync.cfg]
+<= template-sync-download-base
+md5sum = 5695134709e117e7de6890a6a32992ce
+filename = sync.cfg.in

stack/sync/instance-sync-server.cfg.jinja2.in

+[buildout]
+parts =
+  instance-sshd
+  instance-sshd-port
+  user-info
+  cron-entry-instance-sync-server
+
+extends = 
+  {{ template_logrotate_base }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+bin = ${:home}/bin
+etc = ${:home}/etc
+service = ${:etc}/service
+script = ${:etc}/run
+var = ${:home}/var
+log = ${:var}/log
+
+[sync-parameter-dict]
+# sources-etc =
+#  ${directory:etc}/* 
+#  ...
+# destination-etc = ./etc/
+# excludes-etc =
+#  ${directory:etc}/run/* 
+#  ${directory:etc}/service/* 
+folder-section-list = 
+#  etc
+
+[sync-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_sync_cfg }}
+rendered = ${authenticated-server-parameters:web-directory}/sync.cfg
+context =
+  raw ssh_user ${user-info:pw-name}
+  raw ssh_host ${instance-sshd-port:ip}
+  raw ssh_port ${instance-sshd-port:port}
+  raw use_ipv6 True
+  raw rsync_exec {{ rsync_executable_location }}
+  section directory_dict sync-parameter-dict
+
+[before-server-ready-script]
+recipe = collective.recipe.template
+start-file = ${directory:var}/server-is-ready
+input = inline:#!/bin/sh
+  touch ${:start-file}
+  exit 0
+
+output = ${directory:bin}/sync-server-pre-ready
+mode = 700
+
+# to prevent override instance which is deployed by creating clone on it, only set this file
+[instance-is-deployed]
+recipe = plone.recipe.command
+output = ${directory:var}/instance-is-deployed
+command = 
+  touch ${:output}
+stop-on-error = true
+update-command = ${:command}
+
+[server-post-sync-script]
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  echo "Data successfully send to client instance."
+  exit 0
+
+output = ${directory:bin}/server-post-sync
+mode = 700
+
+[logrotate-sync-server]
+< = logrotate-entry-base
+name = instance-sync-server
+log = ${directory:log}/prepare_sync.log
+
+[server-authorized-key]
+recipe = cns.recipe.symlink
+file = ${instance-sshd:public-key}
+symlink = 
+  ${:file} = ${authenticated-server-parameters:web-directory}/authorized_key
+
+[instance-sync-server]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_server_sync_sh }}
+rendered = ${directory:bin}/prepare-instance-sync
+server-ssh-key = ${server-authorized-key:file}
+server-pre-ready-script = ${before-server-ready-script:output}
+server-ready-file = ${before-server-ready-script:start-file}
+server-post-sync-script = ${server-post-sync-script:output}
+context =
+  raw output_log_file ${logrotate-sync-server:log}
+  raw bash_executable_location {{ bash_executable_location }}
+  key server_directory authenticated-server-parameters:web-directory
+  key authorized_key_file instance-sshd:authorized-key-file
+  key server_pre_ready_script :server-pre-ready-script
+  key server_ready_file :server-ready-file
+  key server_post_sync_script :server-post-sync-script
+  key sync_cfg sync-conf:rendered
+  key instance_is_deployed instance-is-deployed:output
+mode = 700
+
+[cron-entry-instance-sync-server]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = instance-sync-server
+frequency = * * * * *
+command = ${instance-sync-server:rendered}
+

stack/sync/instance-sync.cfg.jinja2.in

+[buildout]
+parts =
+  instance-sync-wrapper
+extends = 
+  {{ template_logrotate_base }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+bin = ${:home}/bin
+etc = ${:home}/etc
+service = ${:etc}/service
+script = ${:etc}/run
+var = ${:home}/var
+log = ${:var}/log
+ssl = ${:etc}/ssl
+ssh = ${:etc}/ssh
+ssl-ca = ${:etc}/ssl-request
+sshkeys = ${:srv}/sshkeys
+client-requests = ${:sshkeys}/client-requests
+client-keys = ${:sshkeys}/client-keys
+ssh-hidden = ${buildout:directory}/.ssh
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
+tap-ipv4 = 
+tap-gateway =
+tap-netmask = 
+tap-network = 
+
+[client-configuration]
+recipe = collective.recipe.template
+input = inline:[client]
+  title = ${slap-configuration:instance-title}
+  root-title = ${slap-configuration:root-instance-title}
+  local-ipv4 = ${slap-configuration:ipv4-random}
+  local-ipv6 = ${slap-configuration:ipv6-random}
+  tap-ipv4 = ${slap-configuration:tap-ipv4}
+  tap-gateway = ${slap-configuration:tap-gateway}
+  tap-netmask = ${slap-configuration:tap-netmask}
+  tap-network = ${slap-configuration:tap-network}
+
+output = ${directory:etc}/instance-clone-info
+mode = 600
+
+# client certificate
+[certificate-client-request]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/request-client-certificate
+cert-file = ${directory:ssl}/instance.cert.pem
+key-file = ${directory:ssl}/instance.key.pem
+ca-cert = ${directory:ssl}/cacert.pem
+common-name = instance@${slap-configuration:instance-title}
+command-line = {{ certificate_request_bin }}
+  --cert_file ${:cert-file}
+  --key_file ${:key-file}
+  --cn ${:common-name}
+  --ca_url ${slap-parameter:ca-url}
+  --ca_cert_file ${:ca-cert}
+
+[instance-sync]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_sync_sh }}
+rendered = ${directory:bin}/instance-sync-raw
+context =
+  key ca_certificate certificate-client-request:ca-cert
+  key client_base_dir buildout:directory
+  key client_certificate certificate-client-request:cert-file
+  key client_key certificate-client-request:key-file
+  key server_url slap-parameter:instance-server-url
+  key client_info client-configuration:output
+  raw bash_executable_location {{ bash_executable_location }}
+  raw curl_executable_location {{ curl_executable_location }}
+  raw client_identity_file ${openssh-client:public-key}
+  raw exit_completed_file ${directory:srv}/.instance_sync_done
+  raw output_log_file ${logrotate-sync-client:log}
+  raw python_rsync_script {{ template_python_sync }}
+  raw rsync_executable_location {{ rsync_executable_location }}
+  raw ssh_client_location ${openssh-client:wrapper}
+  raw ssh_known_hosts_file ${buildout:directory}/.ssh/known_hosts
+  raw instance_post_sync_script ${instance-post-sync-script:output}
+  raw client_name ${slap-configuration:instance-title}
+mode = 700
+depends =
+  ${promse-instance-cloned:output}
+
+[promse-instance-cloned]
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  if [ ! -s "$CLONE_DIR/ID" ]; then
+    echo "Sync not started yet"
+    exit 1
+  fi
+  if [ -s "${directory:srv}/.instance_sync_done" ]; then
+    echo "Sync is finished."
+  else
+    echo "Sync is not finished."
+    exit 2
+  fi
+
+output = ${directory:promises}/promise-instance-is-cloned
+mode = 700
+
+[instance-post-sync-script]
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  echo "Sync finished."
+  exit 0
+
+output = ${directory:bin}/instance-post-sync
+mode = 700
+
+[logrotate-sync-client]
+< = logrotate-entry-base
+name = instance-sync-client
+log = ${directory:log}/instance-sync.log
+
+[instance-sync-wrapper]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/instance-sync
+command-line = ${instance-sync:rendered}
+depends = 
+  ${client-sshkeys-authority:wrapper}
+  ${openssh-client:wrapper}
+wait-for-files =
+  ${certificate-client-request:ca-cert}
+  ${certificate-client-request:cert-file}
+  ${certificate-client-request:key-file}
+
+[client-sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${directory:client-requests}
+keys-directory = ${directory:client-keys}
+wrapper = ${directory:services}/client_sshkeys_authority
+keygen-binary = {{ openssh_location }}/bin/ssh-keygen
+
+[openssh-client-raw]
+recipe = slapos.cookbook:wrapper
+command-line = {{ openssh_location }}/bin/ssh -T -o "UserKnownHostsFile ${buildout:directory}/.ssh/known_hosts" -i ${:identity-file}
+wrapper-path = ${directory:bin}/ssh_raw
+identity-file = ${directory:ssh}/client.id_rsa
+parameters-extra = true
+
+[openssh-client]
+<= client-sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = ssh-client
+type = rsa
+executable = ${openssh-client-raw:wrapper-path}
+public-key = ${openssh-client-raw:identity-file}.pub
+private-key = ${openssh-client-raw:identity-file}
+wrapper = ${directory:bin}/ssh
+
+[slap-parameter]
+instance-server-url = 
+ca-url = 

stack/sync/template-instance-clone.cfg.jinja2.in

+[buildout]
+parts =
+  check-instance-cloned
+  instance-sync-wrapper
+  publish-connection-information
+  monitor-base
+
+extends = 
+  ${template-instance-sync:rendered}
+  {{ template_monitor }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+bin = $${:home}/bin
+script = $${:etc}/run
+var = $${:home}/var
+run = $${:var}/run
+
+[instance-sync-wrapper]
+wrapper-path = $${directory:script}/instance-clone
+
+# Script to run when instance is synced
+[instance-post-sync-script]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/instance-bang
+notify = $${directory:run}/instance-bang-done
+instance-is-deployed = $${directory:var}/instance-is-deployed
+command-line = 
+  ${buildout:directory}/bin/${slapos-cookbook:interpreter} ${template-python-bang:location}/${template-python-bang:filename} --server_url "$${slap-connection:server-url}" --key_file "$${slap-connection:key-file}" --cert_file "$${slap-connection:cert-file}" --computer_guid "$${slap-connection:computer-id}" --partition_id "$${slap-connection:partition-id}" --software_release "$${slap-connection:software-release-url}" --message "Instance clone finished" --output "$${:notify}" --publish_name "$${slap-parameter:publish-parameter-name}"
+output = $${:wrapper-path}
+parameters-extra = true
+
+[check-instance-cloned]
+recipe = plone.recipe.command
+# If the instance is cloned, don't try to clone again
+command = 
+  FORCE_CLONE="$${slap-parameter:force-create-clone}"
+  if [ -f "$${instance-post-sync-script:notify}" ]; then
+    echo "Instance already cloned and should be requested with destination software-type.";
+    exit 1;
+  fi
+  if [ $FORCE_CLONE != "True" ] && [ -f "$${instance-post-sync-script:instance-is-deployed}" ]; then
+    echo "Instance was not requested as clone and might be already deployed. Please check your parameters";
+    exit 1;
+  fi
+update-command = $${:command}
+stop-on-error = true
+
+[publish-connection-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+instance-server-url = $${slap-parameter:instance-server-url}
+ca-url = $${slap-parameter:ca-url}
+
+[slap-parameter]
+instance-server-url = 
+ca-url = 
+ca-certificate =
+# A parameter name to publish on master when instance is cloned
+# Can be used to check is clone is done or not
+# do not publish this parameter in your instance
+publish-parameter-name = clone-is-done
+force-create-clone = False
\ No newline at end of file

stack/sync/template/instance_bang.py

+#!/usr/bin/env python
+
+import logging
+import time
+import traceback
+import argparse
+
+import slapos
+from slapos.slap.slap import NotFoundError
+
+def parseArguments():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--server_url',
+                      help='URL of SlapOS Master.')
+  parser.add_argument('--key_file',
+                      help='Path of key file.')
+  parser.add_argument('--cert_file',
+                      help='Path of certificate file.')
+  parser.add_argument('--computer_guid',
+                      help='Computer GUID (ex: COMP1234).')
+  parser.add_argument('--partition_id',
+                      help='Partition ID (ex slappart12).')
+  parser.add_argument('--software_release',
+                      help='Software release URL.')
+  parser.add_argument('--partition_reference',
+                      default=None,
+                      help='The name of the instance.')
+  parser.add_argument('--software_type',
+                      default=None,
+                      help='Instance software type.')
+  parser.add_argument('--state',
+                      default='started',
+                      help='Instance state.')
+  parser.add_argument('--message',
+                      default='Instance Bang',
+                      help='Bang message.')
+  parser.add_argument('--output',
+                      help='The path of file to create at end with output content')
+  parser.add_argument('--publish_name',
+                      default=None,
+                      help='Connection parameter to publish.')
+  parser.add_argument('--instance_guid',
+                      default=None,
+                      help='If request slave, Root instance guid.')
+  parser.add_argument('--parameter',
+                      default=[],
+                      action='append',
+                      help='List of parameters to pass to the request method.')
+  parser.add_argument('--return',
+                      default=[],
+                      action='append',
+                      dest='return_list',
+                      help='List of return connection parameters.')
+  return parser.parse_args()
+
+def bang(server_url, key_file, cert_file, computer_guid, partition_id,
+        software_release, partition_reference=None, software_type='default',
+        state='started', message="Instance bang", publish_option_name=None,
+        instance_guid=None, parameter=[], return_list=[]):
+
+  logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                      stream=sys.stdout ,level=logging.DEBUG)
+
+  slap = slapos.slap.slap()
+  slap.initializeConnection(server_url, key_file, cert_file)
+  current_partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                     partition_id=partition_id)
+
+  if partition_reference is not None:
+    request_dict = dict(state=state)
+    logging.info("requesting %s with software-type %s" % (partition_reference,
+                                                             software_type))
+    if instance_guid:
+      # request instance as slave
+      request_dict['shared'] = True
+      request_dict['filter_kw'] = {'instance_guid': instance_guid}
+    if parameter:
+      param_dict = [v.split('=') for v in parameter]
+      request_dict['partition_parameter_kw'] = {v[0]: v[1] for v in param_dict}
+    # request the instance
+    instance = current_partition.request(software_release,
+                              software_type,
+                              partition_reference,
+                              **request_dict)
+    if return_list:
+      for param in return_list:
+        try:
+          instance.getConnectionParameter(param)
+        except NotFoundError:
+          logging.error("Instance connection parameter %s not found!" % param)
+          return False
+
+  elif publish_option_name:
+    publish_dict = current_partition.getConnectionParameterDict().copy()
+    logging.info("Publishing connection parameter '%s' to master..." % publish_option_name)
+    publish_dict[publish_option_name] = 'OK'
+    current_partition.setConnectionDict(publish_dict)
+
+  current_partition.bang(message=message)
+  logging.info("Bang called on computer partition %s." % partition_id)
+  return True
+
+if __name__ == "__main__":
+  parser = parseArguments()
+  result = bang(parser.server_url, parser.key_file, parser.cert_file, parser.computer_guid,
+        parser.partition_id, parser.software_release, parser.partition_reference,
+        parser.software_type, parser.state, parser.message, parser.publish_name,
+        instance_guid=parser.instance_guid, parameter=parser.parameter,
+        return_list=parser.return_list)
+  if result:
+    with open(parser.output, 'w') as f:
+      f.write("%s banged." % parser.partition_id)

stack/sync/template/prepare-sync-server.sh.in

+#!{{ bash_executable_location }}
+
+set -e
+LC_ALL=C
+export LC_ALL
+umask 077
+
+# Redirect output to log
+exec > >(tee -ai {{ output_log_file }})
+exec 2>&1
+
+BASE_WEB_FOLDER={{ server_directory }}
+AUTHORIZED_KEY_FILE={{ authorized_key_file }}
+BEFORE_READY_SCRIPT={{ server_pre_ready_script }}
+POST_SCRIPT={{ server_post_sync_script }}
+READY_FILE_COND={{ server_ready_file }}
+# Before add client ssh key the server will check if READY_FILE_COND exists. If not
+# he will execute BEFORE_READY_SCRIPT. If no BEFORE_READY_SCRIPT is specified,
+# client key will be added.
+
+ID_FOLDER=$BASE_WEB_FOLDER/ids
+KEYS_FOLDER=$BASE_WEB_FOLDER/keys
+
+mkdir -p $ID_FOLDER $KEYS_FOLDER
+
+# Allow client to start sync
+for ID in `ls $ID_FOLDER`; do
+  if [ -s "$ID_FOLDER/$ID" ]; then
+    READY_KEY=$KEYS_FOLDER/$ID"_ready"
+    DONE_KEY=$KEYS_FOLDER/$ID"_done"
+    OK_KEY=$KEYS_FOLDER/$ID"_OK"
+    name=$(cat $ID_FOLDER/$ID)
+  
+    if [ ! -z "$BEFORE_READY_SCRIPT" ] && [ ! -f "$READY_FILE_COND" ]; then
+      $BEFORE_READY_SCRIPT $ID $name
+    fi
+
+    if [ ! -z "$BEFORE_READY_SCRIPT" ] && [ ! -f "$READY_FILE_COND" ]; then
+      # Condition is not validated
+      echo "Server is not ready yet waiting for $READY_FILE_COND -> [$name]"
+      rm -f $READY_KEY
+      continue
+    fi
+    touch $READY_KEY
+  else
+    echo "WARNING: bad or empty ID file sulied at $ID_FOLDER/$ID..."
+    continue
+  fi
+  if [ -f "$READY_KEY" ] && [ ! -f "$OK_KEY" ] ; then
+    SSH_KEY=$KEYS_FOLDER/$ID"_key"
+
+    # check if authorized_key is added
+    if [ -s "$SSH_KEY" ]; then
+      CLIENT_KEY=$(head -1 $SSH_KEY)
+      R=0
+      grep -q -F "$CLIENT_KEY" $AUTHORIZED_KEY_FILE || R=$?
+      if [ $R -ne 0 ]; then
+        echo "[$(date)] Adding ssh public key from remote client $name"
+        echo "$CLIENT_KEY" >> $AUTHORIZED_KEY_FILE;
+      fi
+    fi
+
+    if [ -f "$DONE_KEY" ] && [ ! -f "$OK_KEY" ]; then
+      # Executing post sync script
+      if [ ! -z $POST_SCRIPT ]; then
+        echo "Executing $POST_SCRIPT with $ID and $name..."
+        RESULT=0
+        $POST_SCRIPT $ID $name || RESULT=$?
+        if [ $R -ne 0 ]; then
+          echo "[ERROR]($ID - $name) Post sync script exited with non zero return code: $RESULT."
+          continue
+        fi
+      fi
+      echo "[$(date)] Instance successfully synced with client $name."
+      echo "$(date)" > $OK_KEY
+    fi
+  fi
+done

stack/sync/template/python_rsync.py

+#!/usr/bin/env python
+
+import os
+import sys
+import subprocess
+import json
+import ConfigParser
+import traceback
+import argparse
+import errno
+import logging
+
+"""
+** Sample config.cfg **
+[sync]
+ssh-user = user
+ssh-host = host
+ssh-port = 1234
+rsync-command = rsync_path
+sync-section-list = 
+  toto
+  sample
+
+[toto]
+source-list = srv/virtual.qcow2
+exclude-list = 
+destination = srv/
+
+[sample]
+source-list = 
+  etc/ssl/
+  etc/config/
+exclude-list = 
+  etc/ssl/xxx.pem
+  etc/ssl/.tmp/*
+destination = etc/
+ignore-missing-source = true|false
+"""
+
+def parseArguments():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--rsync_bin',
+                      help='Path of rsync excutable.')
+  parser.add_argument('--ssh_client_bin',
+                      help='Path of ssh client to use by rsync.')
+  parser.add_argument('--working_dir',
+                      help='Current work directory.')
+  parser.add_argument('--host_config',
+                      help='Remote server directory/ssh definition file.')
+
+  return parser.parse_args()
+
+def executeCommand(args, shell=False):
+    return subprocess.Popen(
+      args,
+      stdin=None,
+      stdout=None,
+      stderr=subprocess.STDOUT,
+      shell=shell
+    )
+
+def mkdir_p(path):
+    try:
+      os.makedirs(path)
+    except OSError as exc:
+      if exc.errno == errno.EEXIST and os.path.isdir(path):
+        pass
+      else:
+        raise
+
+def parse_config(definition_file):
+  config = ConfigParser.ConfigParser()
+  try:
+    config.read(definition_file)
+  except ConfigParser.MissingSectionHeaderError:
+    traceback.print_exc()
+
+  config_dict = {}
+  config_dict['ssh-host'] = config.get('sync', 'ssh-host').strip()
+  config_dict['ssh-port'] = config.get('sync', 'ssh-port').strip()
+  config_dict['ssh-user'] = config.get('sync', 'ssh-user').strip()
+  config_dict['ipv6'] = config.get('sync', 'ipv6').strip()
+  config_dict['rsync-command'] = config.get('sync', 'rsync-command').strip()
+  config_dict['files'] = []
+  sync_section_list = config.get('sync', 'sync-section-list').split()
+
+  for section in sync_section_list:
+    if section:
+      tmp_dict = {}
+      tmp_dict['source'] = [x.strip() for x in config.get(section, 'source-list').split() if x]
+      tmp_dict['exclude'] = [x for x in config.get(section, 'exclude-list').split() if x]
+      tmp_dict['destination'] = config.get(section, 'destination').strip()
+      tmp_dict['ignore-missing-source'] = config.get(section, 'ignore-missing-source').strip()
+      tmp_dict['follow-symlink'] = config.get(section, 'follow-symlink').strip()
+      config_dict["files"].append(tmp_dict)
+
+  if not config_dict["files"]:
+    raise ValueError("Nothing to sync: sync-section-list should contain list of describing folders/files to sync")
+  print config_dict
+  return config_dict
+
+def run_rsync(parser):
+  logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                      stream=sys.stdout ,level=logging.DEBUG)
+  config_dict = parse_config(parser.host_config)
+  os.chdir(parser.working_dir)
+
+  cmd_list_base = [parser.rsync_bin, '-avz', '--stats', '-e', '"%s -p %s"' % (
+                    parser.ssh_client_bin, config_dict['ssh-port']
+                  ), '--rsync-path', config_dict['rsync-command'], '--delete']
+
+  if config_dict['ipv6'] in ['True', 'true', '1']:
+    cmd_list_base.append('-6')
+    config_dict['ssh-host'] = '\[%s\]' % config_dict['ssh-host']
+
+  for config in config_dict['files']:
+    cmd_list = [] + cmd_list_base
+    if config['ignore-missing-source'] in ['true', 'yes', '1']:
+      cmd_list.append('--ignore-missing-args')
+    if config['follow-symlink'] in ['true', 'yes', '1']:
+      cmd_list.append('-L')
+    for path in config['exclude']:
+      cmd_list.extend(['--exclude', path])
+    for path in config['source']:
+      cmd_list.append('%s@%s:%s' % (config_dict['ssh-user'],
+                                    config_dict['ssh-host'],
+                                    path))
+    cmd_list.append(config['destination'])
+    #mkdir_p(config['destination'])
+    
+    logging.info("Executing command: %s" % ' '.join(cmd_list))
+    #process = executeCommand(cmd_list)
+    # XXX Use shell=True
+    process = executeCommand(' '.join(cmd_list), shell=True)
+    process.communicate()
+    if process.returncode != 0:
+      raise Exception("ERROR: rsync command exited with non 0 return code.")
+    logging.info("All rsync command succeeded.")
+
+if __name__ == "__main__":
+  parser = parseArguments()
+  run_rsync(parser)
+

stack/sync/template/sync.cfg.in

+{% set directory_section_list = directory_dict['folder-section-list'].split('\n') -%}
+[sync]
+ssh-user = {{ ssh_user }}
+ssh-host = {{ ssh_host }}
+ssh-port = {{ ssh_port }}
+ipv6 = {{ use_ipv6 }}
+rsync-command = {{ rsync_exec }}
+sync-section-list = {{ directory_section_list | join('\n  ') }}
+
+{% for section in directory_section_list -%}
+{%   if directory_dict.has_key('sources-'~section) -%}
+
+[{{ section }}]
+source-list =  {{ directory_dict['sources-'~section].split('\n') | join('\n  ') }}
+exclude-list = {{ directory_dict['excludes-'~section].split('\n') | join('\n  ') }}
+destination = {{  directory_dict['destination-'~section] }}
+{%     if directory_dict.get('ignore-missing-'~section) in ['true', 'yes', 1] -%}
+ignore-missing-source = true
+{%     else -%}
+ignore-missing-source = false
+{%     endif -%}
+{%     if directory_dict.get('follow-symlink-'~section, '') in ['true', 'yes', 1] -%}
+follow-symlink = true
+{%     else -%}
+follow-symlink = false
+{%     endif -%}
+{%   endif -%}
+{% endfor -%}
\ No newline at end of file

stack/sync/template/template-sync.sh.in

+#! {{ bash_executable_location }}
+
+set -e
+LC_ALL=C
+export LC_ALL
+umask 077
+
+# Redirect output to log
+exec > >(tee -ai {{ output_log_file }})
+exec 2>&1
+NAME={{ client_name }}
+BASE_DIR={{ client_base_dir }}
+POST_SCRIPT={{ instance_post_sync_script }}
+KNOWN_HOSTS_FILE={{ ssh_known_hosts_file }}
+CLONE_DIR=$BASE_DIR/var/clone-op
+mkdir -p $CLONE_DIR
+
+if [ -s "$CLONE_DIR/ID" ]; then
+  UNIQUE_KEY=$(cat $CLONE_DIR/ID)
+else
+  UNIQUE_KEY=$(date +%s%N)
+  echo $UNIQUE_KEY > $CLONE_DIR/ID
+fi
+
+AUTHORIZED_KEY=$UNIQUE_KEY"_key"
+READY_KEY=$UNIQUE_KEY"_ready"
+DONE_KEY=$UNIQUE_KEY"_done"
+INFO_KEY=$UNIQUE_KEY"_info"
+
+# Reinitialise folder
+#if [ -d "$CLONE_DIR" ]; then
+#  rm -rf $CLONE_DIR;
+#fi
+if [ -f "{{ exit_completed_file }}" ]; then
+  rm -f {{ exit_completed_file }};
+fi
+
+curl_client () {
+  # -w %{http_code}
+  {{ curl_executable_location }} -k -L --cert {{ client_certificate }} --key {{ client_key }} --cacert {{ ca_certificate }} $*
+}
+
+curl_cmd () {
+  RET=$({{ curl_executable_location }} -k -L --cert {{ client_certificate }} --key {{ client_key }} --cacert {{ ca_certificate }} -s -w %{http_code} $*)
+  if [ "$RET" -eq 200 ] || [ "$RET" -eq 304 ] || [ "$RET" -eq 201 ]  || [ "$RET" -eq 204 ]; then
+    echo "[SUCCESS $RET]: curl -k -sL $*"
+  else
+    echo "[ERROR $RET]: Failed to execute command curl -k -sL $*"
+    exit 1
+  fi
+}
+
+echo -e "\n\nStart copy of data at : $(date), with ID=$UNIQUE_KEY"
+
+# Check if client can connect to the server
+echo "Connecting to server instance..."
+curl_cmd -o /dev/null {{ server_url }}
+
+cp -f {{ client_identity_file }} $CLONE_DIR/$AUTHORIZED_KEY
+cp -f {{ client_info }} $CLONE_DIR/$INFO_KEY
+
+# send instance ID and ssh public key to the server
+curl_cmd -X PUT -d "$NAME" -o /dev/null {{ server_url }}/ids/$UNIQUE_KEY
+curl_cmd -T $CLONE_DIR/$AUTHORIZED_KEY -o /dev/null {{ server_url }}/keys/
+curl_cmd -T $CLONE_DIR/$INFO_KEY -o /dev/null {{ server_url }}/keys/
+
+# wait until the server is ready (ssh_key is added to the server)
+CODE=0
+while : ; do
+  CODE=$(curl_client -s -w %{http_code} -o /dev/null {{ server_url }}/keys/$READY_KEY)
+  if [ "$CODE" -eq 200 ]; then
+    echo "[$(date)] Server is ready to start sync of data..."
+    break
+  fi
+  echo "Server returned Code $CODE while getting file at {{ server_url }}/keys/$READY_KEY ..."
+  sleep 30
+done
+
+# Get list of file to sync
+curl_cmd -o $CLONE_DIR/server-sync.cfg {{ server_url }}/sync.cfg
+
+# Get Server ssh authorized_key
+curl_cmd -o $CLONE_DIR/server-authorized_key {{ server_url }}/authorized_key
+SSH_HOST=$(cat $CLONE_DIR/server-sync.cfg | grep ssh-host | cut -d ' ' -f3)
+SSH_PORT=$(cat $CLONE_DIR/server-sync.cfg | grep ssh-port | cut -d ' ' -f3)
+R=0
+grep -q -F "[$SSH_HOST]:$SSH_PORT" $KNOWN_HOSTS_FILE || R=$?
+if [ $R -ne 0 ]; then
+  echo "[$(date)] Adding [$SSH_HOST]:$SSH_PORT to known hosts..."
+  echo "[$SSH_HOST]:$SSH_PORT $(cat $CLONE_DIR/server-authorized_key)" >> $KNOWN_HOSTS_FILE;
+fi
+
+# Sync files
+echo "Start sync data..."
+{{ python_rsync_script }} --rsync_bin={{ rsync_executable_location }} --ssh_client_bin={{ ssh_client_location }} --working_dir=$BASE_DIR --host_config=$CLONE_DIR/server-sync.cfg
+
+# Sync is finished, notify the server
+touch $CLONE_DIR/$DONE_KEY
+curl_cmd -T $CLONE_DIR/$DONE_KEY -o /dev/null {{ server_url }}/keys/
+sleep 80
+
+# Execute post sync script
+{{ bash_executable_location }} $POST_SCRIPT
+
+echo 0 > {{ exit_completed_file }}
+echo "Sync finished successfully."