NEO: add support for reflink tool (GC)

See neoppod@f4a5fe3b.

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = b4baf7f21f450fa522c2a69f5a4aedf7
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = eefc3358852b7509bfe1b18da19abca0
+md5sum = 1320dbea362ce6909f1490ba349c3e6c
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 53b2c73a0c5c7cb5f6b841ba6892fa46
+md5sum = d8a62f4a2b68fe97146871d07a500443
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-neo-input-schema.json

@@ -28,6 +28,49 @@
           "description": "Master nodes in the cluster to backup.",
           "type": "string"
         },
+        "reflink": {
+          "description": "Track references between OIDs of an external ZODB.",
+          "additionalProperties": false,
+          "required": [
+            "zurl"
+          ],
+          "properties": {
+            "zurl": {
+              "description": "ZODB to track.",
+              "type": "string"
+            },
+            "_ca": {
+              "type": "string"
+            },
+            "_cert": {
+              "type": "string"
+            },
+            "_key": {
+              "type": "string"
+            },
+            "max-txn-size": {
+              "description": "Maximum number of OIDs to delete per transaction.",
+              "default": 2097151,
+              "type": "integer"
+            },
+            "commit-interval": {
+              "description": "Commit every SECONDS of work.",
+              "default": 10,
+              "type": "integer"
+            },
+            "period": {
+              "description": "Age of the historical revision at which a GC is performed (this can be seen as a grace period). See --period option.",
+              "default": 86400,
+              "type": "integer"
+            },
+            "no-gc": {
+              "description": "Only track references.",
+              "default": false,
+              "type": "boolean"
+            },
+          },
+          "type": "object"
+        },
         "monitor": {
           "description": "Parameters for monitoring.",
           "properties": {
@@ -124,7 +167,7 @@
               "type": "array"
             },
             "storage-type": {
-              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
+              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite. SQLite is recommended for a reflink DB.",
               "enum": [
                 "MySQL",
                 "SQLite"

software/neoppod/instance-neo.cfg.in

@@ -194,6 +194,44 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
+{% set reflink = slapparameter_dict.get('reflink') -%}
+{% if reflink -%}
+{%   set zurl = 'neo://%s@%s' % (slapparameter_dict['cluster'], slapparameter_dict['masters'].replace(' ', ',')) -%}
+{%   if slapparameter_dict['ssl'] -%}
+{%     set zurl = zurl + '?ca=${ca.crt:output}&cert=${neo.crt:output}&key=${neo.key:output}' -%}
+{%   endif -%}
+[{{ section('reflink') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/reflink
+{%   set args = [] -%}
+{%   if 'commit-interval' in reflink -%}
+{%     do args.extend(('-i', str(reflink['commit-interval']))) -%}
+{%   endif -%}
+{%   if 'period' in reflink -%}
+{%     do args.extend(('-p', str(reflink['period']))) -%}
+{%   endif -%}
+{%   if 'no-gc' in reflink -%}
+{%     do args.append('--no-gc') -%}
+{%   endif -%}
+command-line = {{ bin_directory }}/reflink -v {{ zurl }} run {{ ' '.join(args) }} {{ reflink['zurl'] }}
+{%   if reflink['zurl'].startswith('neos://') -%}
+environment =
+  NEO_CA=${reflink-ca:output}
+  NEO_CERT=${reflink-cert:output}
+  NEO_KEY=${reflink-key:output}
+
+{%     for x in 'ca', 'cert', 'key' -%}
+[reflink-{{x}}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/${:_buildout_section_name_}.pem
+inline =
+  {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(reflink['_'+x])}}
+{%     endfor -%}
+{%   endif -%}
+{% endif -%}
+
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template

software/neoppod/root-common.cfg.in

@@ -133,6 +133,9 @@ config-monitor-port = {{ dumps(port) }}
 {%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
+{%- if 'reflink' in parameter_dict %}
+config-reflink = {{ dumps(parameter_dict.pop('reflink')) }}
+{%- endif %}
 {%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}

stack/erp5-zope2/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 6d9a926e07b674ffdaecd381d763c068
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6b10ab0c54278156caf058ebb7246645
+md5sum = 7ac187ff9fe306297724a730459e1a4d
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5-zope2/instance-erp5.cfg.in

@@ -186,6 +186,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 8e452bd32fc0d4d858b275a2b3ee790b
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 1333d2fc21f64da4010a4eafea59d141
+md5sum = bd8cb060b37d8dc6a1ab4e25a2a7e109
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -183,6 +183,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}