NEO: review schema and add support for SQLite configuration

See neoppod@71564067

software/neoppod/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = e000e7134113b9d1c63d40861eaf0489
+md5sum = ecc98da90cd446ea224ddeece1374190
 
 [root-common]
 filename = root-common.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = fda911d5ef9efee365f1b0ff9843a50b
+md5sum = c3fdf8c6aae89acd42d9c1ea052dfb7d
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-common.cfg.in

@@ -41,6 +41,7 @@ url = {{ neo_master }}
 <= jinja2-template-base
 url = {{ neo }}
 extra-context =
+    import urllib urllib
     key master_cfg neo-master:output
     key admin_cfg neo-admin:output
 {%- if mariadb_location is defined %}

software/neoppod/instance-neo-input-schema.json

@@ -110,14 +110,6 @@
               "default": false,
               "type": "boolean"
             },
-            "storage-type": {
-              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
-              "enum": [
-                "MySQL",
-                "SQLite"
-              ],
-              "type": "string"
-            },
             "private-tmpfs": {
               "description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
               "type": "string"
@@ -126,27 +118,58 @@
               "description": "List of bindings to test when running the test suite.",
               "type": "array"
             },
+            "sqlite": {
+              "description": "Storage backend configuration. This selects the SQLite backend.",
+              "properties": {
+                "relaxed-writes": {
+                  "description": "When enabled, sets synchronous = OFF and journal_mode = MEMORY - RTFM, those options are dangerous.",
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": {
+                "description": "See NEO documentation for the list of supported settings.",
+                "type": ["number", "string"]
+              },
+              "type": "object"
+            },
             "mysql": {
-              "description": "Dictionary containing parameters for MySQL.",
-              "default": {},
+              "description": "MariaDB server configuration. This selects the MySQL backend.",
               "properties": {
                 "relaxed-writes": {
-                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
-                  "default": false,
+                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous.",
                   "type": "boolean"
                 }
               },
               "additionalProperties": {
-                "description": "To configure important parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
+                "description": "To configure parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
                 "type": "string"
               },
               "type": "object"
             },
             "engine": {
-              "description": "Configures storage engine, currently only InnoDB and RocksDB are supported. Defaults to NEO's default.",
+              "description": "For NEO, this is a creation-time parameter and it defaults to NEO's default. For mysqld, this sets plugins to load and it defaults to load all.",
+              "enum": [
+                "InnoDB",
+                "RocksDB"
+              ],
               "type": "string"
             }
           },
+          "oneOf": [
+            {
+              "description": "default (MariaDB if built else SQLite) or SQLite",
+              "properties": {
+                "mysql": false,
+                "engine": false
+              }
+            },
+            {
+              "required": ["mysql"],
+              "properties": {
+                "sqlite": false
+              }
+            }
+          ],
           "type": "object"
         },
         "node!": {

software/neoppod/instance-neo.cfg.in

 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set part_list = [] -%}
 {% set init_list = [] -%}
+{% set directory_dict = {} -%}
 
 {% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
 {% set storage_count = slapparameter_dict.get('storage-count', 1) -%}
+{# BBB: storage-type -#}
+{% if 'mysql' in slapparameter_dict -%}
+{%   do assert('sqlite' not in slapparameter_dict) -%}
+{%   set storage_type = 'MySQL' -%}
+{%   set extra_dict = slapparameter_dict.pop('mysql') -%}
+{% elif 'sqlite' in slapparameter_dict -%}
+{%   set storage_type = 'SQLite' -%}
+{%   set extra_dict = slapparameter_dict.pop('sqlite') -%}
+{% elif storage_count -%}
 {%   set storage_type = slapparameter_dict.get('storage-type') or (
        'MySQL' if mariadb_location is defined else 'SQLite') -%}
-{% set mysql = storage_count and storage_type != 'SQLite' -%}
+{%   set extra_dict = {} -%}
+{% else -%}
+{%   set storage_type = '' -%}
+{% endif -%}
+{% do assert(storage_count or not storage_type) -%}
+{% do assert(slapparameter_dict.get('storage-type', storage_type) == storage_type) -%}
+
+{% set mysql = storage_type == 'MySQL' -%}
 {% if mysql -%}
 
 [{{ section('mysqld') }}]
@@ -37,7 +54,7 @@ tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log
-extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }}
+extra-dict = {{ dumps(extra_dict) }}
 init-file = ${init-script:output}
 engine = {{ slapparameter_dict.get('engine', '') }}
 
@@ -54,6 +71,14 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn
 wrapper-path = ${directory:bin}/${:command}
 command = mysql
 
+{% elif storage_type == 'SQLite' -%}
+
+{%   if extra_dict.pop('relaxed-writes', False) -%}
+{%     do extra_dict.setdefault('synchronous', 'OFF') -%}
+{%     do extra_dict.setdefault('journal_mode', 'MEMORY') -%}
+{%   endif -%}
+{%   set query_string = urllib.urlencode(extra_dict) -%}
+
 {% endif -%}
 
 [{{ section('binary-neolog') }}]
@@ -123,14 +148,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
 {%- if mysql %}
 {%-   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') %}
 database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
-{%- elif private_tmpfs %}
+{%- else %}
+{%-   if private_tmpfs %}
 private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
-database-parameters = ${directory:tmp}/db.sqlite
+{%-     set path = '${directory:tmp}/db.sqlite' %}
 {%-   else %}
-database-parameters = ${directory:db-{{i}}}/db.sqlite
-
-[directory]
-db-{{i}} = ${:srv}/{{ storage_id }}
+{%-     set path = '${directory:db-' ~ i ~ '}/db.sqlite' %}
+{%-     do directory_dict.__setitem__('db-' ~ i, '${:srv}/' + storage_id) %}
+{%-   endif %}
+{%-   if query_string %}
+database-parameters = file:{{ path }}?{{ query_string }}
+{%-   else %}
+database-parameters = {{ path }}
+{%-   endif -%}
 {%- endif %}
 
 [{{ section('logrotate-storage-' ~ i) }}]
@@ -141,17 +171,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-bin = ${buildout:directory}/bin
-etc = ${buildout:directory}/etc
-var = ${buildout:directory}/var
-etc_run =  ${:etc}/run
-var_run =  ${:var}/run
-log = ${buildout:directory}/var/log
-tmp = ${buildout:directory}/tmp
-srv = ${buildout:directory}/srv
-
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template
@@ -196,13 +215,26 @@ context =
     key     datadir           my-cnf-parameters:data-directory
     key     results_directory directory:results
 
-[directory]
-results = ${directory:srv}/tests
+{%-    do directory_dict.__setitem__('results', '${directory:srv}/tests') %}
 {%-  endif %}
 {%- endif %}
 
 {%- endif %}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+etc_run =  ${:etc}/run
+var_run =  ${:var}/run
+log = ${buildout:directory}/var/log
+tmp = ${buildout:directory}/tmp
+srv = ${buildout:directory}/srv
+{%- for k, v in directory_dict.iteritems() %}
+{{ k }} = {{ v }}
+{%- endfor %}
+
 [buildout]
 extends =
   {{ logrotate_cfg }}