Commit e354e214 authored by Julien Muchembled's avatar Julien Muchembled

NEO: new private-tmpfs option to run DB & tests in RAM

parent 28ea60be
Pipeline #8202 failed with stage
......@@ -88,7 +88,9 @@ class NeoBaseRecipe(GenericBaseRecipe):
)
args += self._getOptionList()
args += shlex.split(options.get('extra-options', ''))
return self.createWrapper(options['wrapper'], args)
private_tmpfs = self.parsePrivateTmpfs()
kw = {'private_tmpfs': private_tmpfs} if private_tmpfs else {}
return self.createWrapper(options['wrapper'], args, **kw)
def _getBindingAddress(self):
options = self.options
......
......@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
[instance-neo]
filename = instance-neo.cfg.in
md5sum = 574acb0cae9af8ec2af52825fb2436d8
md5sum = 512383220488335ac186013c2ffdc7c1
[template-neo-my-cnf]
filename = my.cnf.in
......@@ -46,4 +46,4 @@ md5sum = 5afd326de385563b5aeac81039f23341
[runTestSuite.in]
_update_hash_filename_ = runTestSuite.in
md5sum = d724ca0d54dda15bba68b8a87f0e4544
md5sum = 7a0d5d259eb7f90fc0421d1264fbe7b5
......@@ -114,6 +114,10 @@
],
"type": "string"
},
"private-tmpfs": {
"description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
"type": "string"
},
"mysql": {
"description": "Dictionary containing parameters for MySQL.",
"default": {},
......
......@@ -2,22 +2,36 @@
{% set part_list = [] -%}
{% set init_list = [] -%}
{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
{% set storage_type = slapparameter_dict.get('storage-type') or (
'MySQL' if mariadb_location is defined else 'SQLite') -%}
{% set mysql = storage_type == 'MySQL' -%}
{% if mysql -%}
[{{ section('mysqld') }}]
{% if private_tmpfs -%}
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:etc_run}/mariadb
private-tmpfs = {{ private_tmpfs }} ${my-cnf-parameters:tmp-directory}
command-line = ${mariadb-ns:rendered}
[mariadb-ns]
rendered = ${directory:bin}/mariadb-ns
{% else -%}
rendered = ${directory:etc_run}/mariadb
{% endif -%}
recipe = slapos.recipe.template:jinja2
template = {{ template_mysqld_wrapper }}
rendered = ${directory:etc_run}/mariadb
context =
key defaults_file my-cnf:rendered
key datadir my-cnf-parameters:data-directory
[my-cnf-parameters]
socket = ${directory:var_run}/mariadb.sock
{% if private_tmpfs -%}
data-directory = ${:tmp-directory}/mariadb
{% else -%}
data-directory = ${directory:srv}/mariadb
{% endif -%}
tmp-directory = ${directory:tmp}
pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
......@@ -105,6 +119,9 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{%- if mysql %}
{%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
{%- elif private_tmpfs %}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
database-parameters = ${directory:tmp}/db.sqlite
{%- else %}
database-parameters = ${directory:db-{{i}}}/db.sqlite
......@@ -148,6 +165,14 @@ post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS"
{% if runTestSuite_in is defined -%}
# bin/runTestSuite to run NEO tests
[{{ section('runTestSuite') }}]
{%- if private_tmpfs %}
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
command-line = ${runTestSuite-ns:rendered}
[runTestSuite-ns]
{%- endif %}
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
template = {{ runTestSuite_in }}
......@@ -157,6 +182,13 @@ context =
section my_cnf_parameters my-cnf-parameters
raw bin_directory {{ bin_directory }}
raw prepend_path {{ mariadb_location }}/bin
{%- if private_tmpfs %}
key datadir my-cnf-parameters:data-directory
key results_directory directory:results
[directory]
results = ${directory:srv}/tests
{%- endif %}
{%- endif %}
{%- endif %}
......
......@@ -2,7 +2,7 @@
"""
Script to run NEO test suite using Nexedi's test node framework.
"""
import argparse, json, os, re, shutil, subprocess, sys, traceback
import argparse, errno, json, os, re, shutil, subprocess, sys, traceback
from erp5.util import taskdistribution
from time import gmtime, sleep, strftime, time
......@@ -130,6 +130,14 @@ def main():
if timeout < time():
raise RuntimeError("MySQL server not started")
sleep(1)
{%- if datadir is defined %}
# fake path for neostorage (getTopologyPath)
try:
os.mkdir({{ repr(datadir) }})
except OSError as e:
if e.errno != errno.EEXIST:
raise
{%- endif %}
with open(os.devnull) as stdin:
p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
......@@ -161,6 +169,13 @@ def main():
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
stderr=stderr,
**status_dict)
{%- if results_directory is defined %}
results = {{ repr(results_directory + '/') }} + adapter
if os.path.exists(results):
shutil.rmtree(results)
shutil.move(temp, results)
{%- endif %}
if __name__ == "__main__":
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment