rdiff-backup: do not depend on buildout python

component/rdiff-backup/buildout.cfg

 [buildout]
 extends =
   ../librsync/buildout.cfg
-  ../coreutils/buildout.cfg
+  ../python-2.7/buildout.cfg
 parts =
-  rdiff-backup-output
+  rdiff-backup-1.0.5
+  rdiff-backup-1.3.4
 
-[rdiff-backup-build]
-recipe = zc.recipe.egg:custom
-egg = rdiff-backup
-patches =
-  ${:_profile_base_location_}/rdiff-backup-1.2.8-librsync-1.0.0.patch#5e54a67845edd6942fcf7359c921e003
+[rdiff-backup-common]
+recipe = slapos.recipe.cmmi
+shared = true
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
-include-dirs =
-  ${librsync:location}/include/
-library-dirs =
-  ${librsync:location}/lib/
-rpath =
-  ${librsync:location}/lib/
-# If you need another release, override this link
-# and pin the egg in the [versions] section of the stack or SR.
-find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
+configure-command = true
+make-binary = true
+post-install =
+  set %(location)s/lib/python2.7
+  %(python)s setup.py build_ext -I${librsync:location}/include -L${librsync:location}/lib -R${librsync:location}/lib build install --prefix=%(location)s --install-lib=$1
+  cd %(location)s/bin
+  sed -i '/^import\b.*\bsys\b/s,$,\nsys.path[0] = '\"$1\", *
+  ./rdiff-backup -V
+python = ${python2.7:executable}
+
+[rdiff-backup-common:python27]
+python = ${buildout:executable}
+
+[rdiff-backup-1.0.5]
+<= rdiff-backup-common
+url = https://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/${:md5sum}/rdiff-backup-1.0.5.tar.gz
+md5sum = fa2a165fa07a94be52c52e3545bc7758
+patches =
+  ${:_profile_base_location_}/rdiff-backup-1.2.8-librsync-1.0.0.patch#5e54a67845edd6942fcf7359c921e003
 
-[rdiff-backup-build-1.3.4]
-<= rdiff-backup-build
+[rdiff-backup-1.3.4]
+<= rdiff-backup-common
 # use our own version
-find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd6.tar.gz
+url = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd6.tar.gz
+md5sum = 06b8df2099baebca6086a315b031fd53
 patches =
   ${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
-
-[rdiff-backup]
-# Scripts only generation part for rdiff-backup
-recipe = zc.recipe.egg
-eggs =
-  ${rdiff-backup-build:egg}
-entry-points =
-  rdiff-backup=rdiff_backup.Main:Main
-arguments = sys.argv[1:]
-
-[rdiff-backup-output]
-# Shared binary location to ease migration
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = ${coreutils-output:test} -x ${:rdiff-backup}
-rdiff-backup = ${buildout:directory}/bin/rdiff-backup

software/backupserver/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 851262d7174da868805cb7c8e1ced7c0
 
 [template-backup-script]
 filename = template-backup-script.sh.in
-md5sum = 4c75f8462eaef2d8fcf24627e5f3575a
+md5sum = 3f3286347a7e271e7bfa66e1a840989b
 
 [template-crontab-line]
 filename = template-crontab-line.in

software/backupserver/software.cfg

@@ -16,7 +16,6 @@ extends =
   ../../stack/monitor/buildout.cfg
 
 parts =
-  rdiff-backup
   dcron
   logrotate
   nginx
@@ -102,7 +101,6 @@ output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-rdiff-backup = 1.0.5+SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 PyRSS2Gen = 1.1

software/backupserver/template-backup-script.sh.in

@@ -13,7 +13,7 @@ ${coreutils-output:echo} "Available only if backup succeed." > $${:statistic_log
 # set -e
 cd $${:datadirectory}
 ${util-linux:location}/bin/flock $${:status_log}.lock \
-  ${rdiff-backup-output:rdiff-backup} \
+  ${rdiff-backup-1.0.5:location}/bin/rdiff-backup \
   $${:exclude_string} \
   --include='$${:include}' \
   --exclude='**' \
@@ -26,7 +26,7 @@ RESULT=$?
 if [ $RESULT -eq 0 ]
 then
   ${coreutils-output:echo} "`${coreutils-output:date} --iso-8601=seconds -u`, $${:statistic_log}, $${:hostname}, backup success" >> $${:status_log}
-  ${findutils-output:find} rdiff-backup-data/ -maxdepth 1 -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average > $${:statistic_log}
+  ${findutils-output:find} rdiff-backup-data/ -maxdepth 1 -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-1.0.5:location}/bin/rdiff-backup --calculate-average > $${:statistic_log}
 else
   ${coreutils-output:echo} "`${coreutils-output:date} --iso-8601=seconds -u`, $${:statistic_log}, $${:hostname}, backup failed" >> $${:status_log}
 fi

software/caddy-frontend/software.cfg

@@ -6,7 +6,6 @@ extends =
   ../../component/caddy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/logrotate/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
   ../../component/trafficserver/buildout.cfg
   ../../component/6tunnel/buildout.cfg
   ../../component/xz-utils/buildout.cfg
@@ -20,7 +19,6 @@ extends =
 parts +=
   caucase-eggs
   template
-  rdiff-backup
   caddyprofiledeps
   kedifa-develop
   kedifa
@@ -234,7 +232,6 @@ ecdsa = 0.13
 gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-rdiff-backup = 1.0.5+SlapOSPatched001
 smmap = 0.9.0
 
 websockify = 0.8.0

software/kvm/software-py3.cfg

@@ -2,9 +2,5 @@
 extends =
   software.cfg
 
-# Ignore these for now
-common-parts -=
-  rdiff-backup
-
 [python]
 part = python3

software/kvm/software.cfg

@@ -23,7 +23,6 @@ common-parts =
   template
 
 # XXX: we have to manually add this for resilience
-  rdiff-backup
   pbs-recipe-egg
 
 

software/kvm/test/test.py

@@ -51,8 +51,6 @@ from slapos.testing.utils import findFreeTCPPort
 
 has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
 skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
-skipIfPython3 = unittest.skipIf(
-  six.PY3, 'rdiff-backup is not compatible with Python 3 yet')
 
 if has_kvm:
   setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -385,7 +383,6 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
   __partition_reference__ = 'akcb'
@@ -432,7 +429,6 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestInstanceResilient(InstanceTestCase, KvmMixin):
   __partition_reference__ = 'ir'
@@ -515,7 +511,6 @@ ir3:sshd-on-watch RUNNING""",
     )
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestAccessResilientAdditional(InstanceTestCase):
   __partition_reference__ = 'ara'
@@ -849,7 +844,6 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     self.assertPromiseFails(self.config_state_promise)
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestBootImageUrlListResilient(TestBootImageUrlList):
   kvm_instance_partition_reference = 'biul2'
@@ -993,7 +987,6 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     )
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
   kvm_instance_partition_reference = 'bius2'
@@ -1222,7 +1215,6 @@ class TestWhitelistFirewallRequest(TestWhitelistFirewall):
     self.assertIn('4.4.4.4', self.content_json)
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestWhitelistFirewallResilient(TestWhitelistFirewall):
   kvm_instance_partition_reference = 'wf2'
@@ -1232,7 +1224,6 @@ class TestWhitelistFirewallResilient(TestWhitelistFirewall):
     return 'kvm-resilient'
 
 
-@skipIfPython3
 @skipUnlessKvm
 class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
   kvm_instance_partition_reference = 'wf2'

software/slaprunner/software-py3.cfg

@@ -2,10 +2,6 @@
 extends =
   software.cfg
 
-# Ignore these for now
-common-parts -=
-  rdiff-backup
-
 [python]
 part = python3
 

software/slaprunner/software.cfg

@@ -45,7 +45,6 @@ common-parts =
   template-slapformat-definition.cfg
   template-slapuser-script
 # XXX: we have to manually add this for resilience
-  rdiff-backup
   pbs-recipe-egg
 
 parts =

stack/cloudooo.cfg

@@ -24,7 +24,6 @@ extends =
   ../component/mesa/buildout.cfg
   ../component/onlyoffice-core/buildout.cfg
   ../component/poppler/buildout.cfg
-  ../component/rdiff-backup/buildout.cfg
   ../component/xorg/buildout.cfg
 
 versions = versions
@@ -60,7 +59,6 @@ parts =
   file
   poppler
 
-  rdiff-backup
   apache
   haproxy
   cloudooo-develop
@@ -77,7 +75,6 @@ PasteScript = 3.2.0
 WSGIUtils = 0.7.2
 WSGIserver = 1.3
 python-magic = 0.4.18
-rdiff-backup = 1.0.5+SlapOSPatched001
 
 # Required by:
 # PasteScript== 3.2.0

stack/erp5/buildout.cfg

@@ -32,7 +32,6 @@ extends =
   ../../component/pycrypto-python/buildout.cfg
   ../../component/pysvn-python/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
   ../../component/scikit-learn/buildout.cfg
   ../../component/scikit-image/buildout.cfg
   ../../component/PyWavelets/buildout.cfg
@@ -64,7 +63,6 @@ extends =
 parts +=
   erp5-util-develop
   slapos-cookbook
-  rdiff-backup
   mroonga-mariadb
   tesseract
   tesseract-eng-traineddata
@@ -624,9 +622,6 @@ zope.dottedname = 4.1.0
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
 Products.PluggableAuthService = 1.9.0
 
-# we are still using this old stable version.
-rdiff-backup = 1.0.5+SlapOSPatched001
-
 # modified version that works fine for buildout installation
 SOAPpy = 0.12.0nxd001
 

stack/resilient/buildout.cfg

@@ -20,7 +20,6 @@ parts =
 
   # needed tools for resiliency
   gzip
-  rdiff-backup
   dash
 
 [pbs-recipe-egg]
@@ -124,12 +123,3 @@ mode = 0644
 [exporter-default-configuration]
 # Time in minutes to wait for the backup to finish
 backup_wait_time = 10
-
-[rdiff-backup]
-eggs =
-  ${rdiff-backup-build-1.3.4:egg}
-
-[versions]
-# 1.3.4nxdX is invalid version string, thus pached version string is not '1.3.4nxdX+SlapOSPatched001'
-# but '1.3.4nxdX-SlapOSPatched001'.
-rdiff-backup = 1.3.4nxd6-SlapOSPatched001

stack/resilient/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [pbsready]
 filename = pbsready.cfg.in
-md5sum = 4b0f914a54c9be5bff2b86f3416f4584
+md5sum = afb54d1aafd5854116028bc63501409b
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
@@ -26,7 +26,7 @@ md5sum = 2b0c71b085cfe8017f28098c160b1f49
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = 382d9b57faac92da3c723760f2210c13
+md5sum = e64e13854332bcc2595df187fcae1203
 
 [template-replicated]
 filename = template-replicated.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -113,7 +113,7 @@ feeds = $${directory:notifier-feeds}
 callbacks-directory = $${directory:notifier-callbacks}
 equeue-socket = $${equeue:socket}
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+rdiffbackup-binary = ${rdiff-backup-1.3.4:location}/bin/rdiff-backup
 sshclient-binary = $${openssh-client:wrapper-path}
 known-hosts = $${directory:dot-ssh}/known_hosts
 promises-directory = $${basedirectory:promises}
@@ -133,7 +133,7 @@ status-item-directory = $${directory:notifier-status-items}
 
 [pbs-resilient-status-feed]
 recipe = slapos.cookbook:wrapper
-command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${monitor-instance-parameter:root-instance-title}-PBS" --link $${pbs:log-url}
+command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${monitor-instance-parameter:root-instance-title}-PBS" --link $${pbs:log-url}
 feed-path = $${directory:monitor-resilient}/pbs-status-rss
 wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
 
@@ -197,7 +197,7 @@ wrapper-path = $${rootdirectory:bin}/ssh
 [resilient-genstatrss-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard-coded Urls
-command-line = ${buildout:directory}/bin/rdiffbackup.genstatrss --output '$${monitor-directory:public}/resilient-feed' --rdiff_backup_data_folder '$${pbs:rdiff-backup-data-folder}' --feed_url '$${monitor-conf-parameters:base-url}/public/resilient-feed'
+command-line = ${buildout:bin-directory}/rdiffbackup.genstatrss --output '$${monitor-directory:public}/resilient-feed' --rdiff_backup_data_folder '$${pbs:rdiff-backup-data-folder}' --feed_url '$${monitor-conf-parameters:base-url}/public/resilient-feed'
 wrapper-path = $${directory:bin}/resilient-genstatrss.py
 
 [pbs-push-history-log]

stack/resilient/pbsready.cfg.in

@@ -103,7 +103,7 @@ recipe = slapos.cookbook:pbs
 client = false
 path = $${directory:backup}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+rdiffbackup-binary = ${rdiff-backup-1.3.4:location}/bin/rdiff-backup
 
 
 #----------------
@@ -141,7 +141,7 @@ context =
 
 [notifier-resilient-status-feed]
 recipe = slapos.cookbook:wrapper
-command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${notifier:status-item-directory} --title "Status feed for $${notifier:instance-root-name}" --link $${notifier:log-url}
+command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --status-item-path $${notifier:status-item-directory} --title "Status feed for $${notifier:instance-root-name}" --link $${notifier:log-url}
 feed-path = $${directory:monitor-resilient}/notifier-status-rss
 wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
 

[Nicolas Wavrant]

Hi @jm,

I can't find any reference to this section rdiff-backup-common:python27. Is it a leftover, or is there some magic involved ?

[Jérome Perrin]

It's a conditional section ( https://www.buildout.org/en/latest/topics/variables-extending-and-substitutions.html#conditional-configuration-sections )

[Nicolas Wavrant]

I see, thanks @jerome !

[Jérome Perrin]

This also has the side effect that diff-backup is no longer installed in several softwares (erp5, cloudooo). It was used in some ad-hoc backup scripts.

edit: I don't mean that it's bad, if theses software release don't use rdiff-backup, they should not install it. We'll use another way for our back scripts.

/cc @romain