Commit 20bf35fc by Alain Takoudjou

Update Release Candidate

2 parents b04d30a2 425ae92c
Showing 109 changed files with 1793 additions and 1154 deletions
Changes
=======
1.0.75 (2018-09-04)
-------------------
* erp5_test: stop using erp5_test recipe
* random: fix password generation with newlines
* erp5testnode: enable password authentication for scalability test system
* pbs: Ignore numerical IDs (UID/GID) when push
* request: add requestoptional.serialised
1.0.65 (2018-06-22)
-------------------
......
[buildout]
extends =
../gcc/buildout.cfg
../numpy/openblas.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
......@@ -41,6 +42,6 @@ setup-eggs =
${numpy:egg}
${scipy:egg}
rpath =
${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64
${gcc:location}/lib
${gcc:location}/lib64
${openblas:location}/lib
......@@ -9,5 +9,5 @@ depends_gitfetch =
[go_github.com_mholt_caddy]
<= go-git-package
go.importpath = github.com/mholt/caddy
repository = https://github.com/mholt/caddy
revision = v0.11.0-11-ge263566673
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v0.11.0-3-g12438f6cff8c15f307631151eb064cec579b7605
......@@ -25,7 +25,7 @@ find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1
[rdiff-backup-build-1.3.4]
<= rdiff-backup-build
# use our own version
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd5.tar.gz
patches =
${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
......
[buildout]
extends =
../gcc/buildout.cfg
../numpy/openblas.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
......@@ -49,6 +50,6 @@ setup-eggs =
${pillow-python:egg}
networkx
rpath =
${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64
${gcc:location}/lib
${gcc:location}/lib64
${openblas:location}/lib
[buildout]
extends =
../gcc/buildout.cfg
../cython/buildout.cfg
../numpy/openblas.cfg
../scipy/buildout.cfg
......@@ -40,6 +41,6 @@ setup-eggs =
${numpy:egg}
${scipy:egg}
rpath =
${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64
${gcc:location}/lib
${gcc:location}/lib64
${openblas:location}/lib
#!/usr/bin/env python
r"""Command-line tool to format software release JSON for slapos.
Inspired by json.tool from python
Usage::
format-json infile outfile
"""
import os
import sys
import json
import collections
def main():
if len(sys.argv) != 3:
raise SystemExit(sys.argv[0] + " infile outfile")
with open(sys.argv[1], 'rb') as infile:
try:
obj = json.load(infile, object_pairs_hook=collections.OrderedDict)
except ValueError, e:
raise SystemExit(e)
with open(sys.argv[2], 'wb') as outfile:
json.dump(obj, outfile, sort_keys=False, indent=2, separators=(',', ': '))
outfile.write('\n')
if __name__ == '__main__':
main()
......@@ -18,7 +18,10 @@
},
"serialisation": {
"description": "How the parameters and results are serialised",
"enum": ["xml", "json-in-xml"],
"enum": [
"xml",
"json-in-xml"
],
"type": "string"
},
"software-type": {
......@@ -44,7 +47,10 @@
},
"serialisation": {
"description": "How the parameters and results are serialised, if different from global setting, required if global setting is not provided",
"enum": ["xml", "json-in-xml"],
"enum": [
"xml",
"json-in-xml"
],
"type": "string"
},
"request": {
......@@ -55,11 +61,11 @@
"description": "URL, relative to Software Release base path, of a json schema for values published by instance of current software type",
"type": "string"
},
"software-type" : {
"software-type": {
"description": "Value to be used as software type instead of the software type id (in order to use multiple diferent forms for the same software type).",
"type": "string"
},
"shared" : {
"shared": {
"description": "Define if the request will request a Slave or Software Instance.",
"default": "false",
"type": "boolean"
......@@ -78,4 +84,3 @@
},
"type": "object"
}
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.66'
version = '1.0.75'
name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n"
......@@ -160,9 +160,11 @@ setup(name=name,
'readline = slapos.recipe.readline:Recipe',
'redis.server = slapos.recipe.redis:Recipe',
'request = slapos.recipe.request:Recipe',
'request.serialised = slapos.recipe.request:Serialised',
'request.serialised = slapos.recipe.request:RequestJSONEncoded',
'request.edge = slapos.recipe.request:RequestEdge',
'requestoptional = slapos.recipe.request:RequestOptional',
'requestoptional.serialised = '
'slapos.recipe.request:RequestOptionalJSONEncoded',
're6stnet.registry = slapos.recipe.re6stnet:Recipe',
'reverseproxy.nginx = slapos.recipe.reverse_proxy_nginx:Recipe',
'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
......@@ -208,5 +210,6 @@ setup(name=name,
tests_require=[
'jsonschema',
'mock',
'testfixtures',
],
)
......@@ -12,4 +12,4 @@
"type": "string"
}
}
}
\ No newline at end of file
}
......@@ -74,6 +74,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
$RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
--restore-as-of now \\
--ignore-numerical-ids \\
--force \\
%(local_dir)s \\
%(remote_dir)s
......
......@@ -264,7 +264,7 @@ class RequestOptional(Recipe):
update = install
class Serialised(Recipe):
class JSONCodec(object):
def _filterForStorage(self, partition_parameter_kw):
return wrap(partition_parameter_kw)
......@@ -274,7 +274,17 @@ class Serialised(Recipe):
except slapmodule.NotFoundError:
return {}
class RequestJSONEncoded(JSONCodec, Recipe):
"""
Like Recipe, but serialised with JSONCodec.
"""
pass
class RequestOptionalJSONEncoded(JSONCodec, RequestOptional):
"""
Like RequestOptional, but serialised with JSONCodec.
"""
pass
CONNECTION_PARAMETER_STRING = 'connection-'
......
{
"id": "http://json-schema.org/draft-04/schema#",
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Core schema meta-schema",
"definitions": {
"schemaArray": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#" }
"id": "http://json-schema.org/draft-04/schema#",
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Core schema meta-schema",
"definitions": {
"schemaArray": {
"type": "array",
"minItems": 1,
"items": {
"$ref": "#"
}
},
"positiveInteger": {
"type": "integer",
"minimum": 0
},
"positiveIntegerDefault0": {
"allOf": [
{
"$ref": "#/definitions/positiveInteger"
},
"positiveInteger": {
"type": "integer",
"minimum": 0
{
"default": 0
}
]
},
"simpleTypes": {
"enum": [
"array",
"boolean",
"integer",
"null",
"number",
"object",
"string"
]
},
"stringArray": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1,
"uniqueItems": true
}
},
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uri"
},
"$schema": {
"type": "string",
"format": "uri"
},
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"default": {},
"multipleOf": {
"type": "number",
"minimum": 0,
"exclusiveMinimum": true
},
"maximum": {
"type": "number"
},
"exclusiveMaximum": {
"type": "boolean",
"default": false
},
"minimum": {
"type": "number"
},
"exclusiveMinimum": {
"type": "boolean",
"default": false
},
"maxLength": {
"$ref": "#/definitions/positiveInteger"
},
"minLength": {
"$ref": "#/definitions/positiveIntegerDefault0"
},
"pattern": {
"type": "string",
"format": "regex"
},
"additionalItems": {
"anyOf": [
{
"type": "boolean"
},
"positiveIntegerDefault0": {
"allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ]
{
"$ref": "#"
}
],
"default": {}
},
"items": {
"anyOf": [
{
"$ref": "#"
},
"simpleTypes": {
"enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ]
{
"$ref": "#/definitions/schemaArray"
}
],
"default": {}
},
"maxItems": {
"$ref": "#/definitions/positiveInteger"
},
"minItems": {
"$ref": "#/definitions/positiveIntegerDefault0"
},
"uniqueItems": {
"type": "boolean",
"default": false
},
"maxProperties": {
"$ref": "#/definitions/positiveInteger"
},
"minProperties": {
"$ref": "#/definitions/positiveIntegerDefault0"
},
"required": {
"$ref": "#/definitions/stringArray"
},
"additionalProperties": {
"anyOf": [
{
"type": "boolean"
},
"stringArray": {
"type": "array",
"items": { "type": "string" },
"minItems": 1,
"uniqueItems": true
{
"$ref": "#"
}
],
"default": {}
},
"definitions": {
"type": "object",
"additionalProperties": {
"$ref": "#"
},
"default": {}
},
"type": "object",
"properties": {
"id": {
"type": "string",
"format": "uri"
},
"$schema": {
"type": "string",
"format": "uri"
},
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"default": {},
"multipleOf": {
"type": "number",
"minimum": 0,
"exclusiveMinimum": true
},
"maximum": {
"type": "number"
},
"exclusiveMaximum": {
"type": "boolean",
"default": false
},
"minimum": {
"type": "number"
},
"exclusiveMinimum": {
"type": "boolean",
"default": false
},
"maxLength": { "$ref": "#/definitions/positiveInteger" },
"minLength": { "$ref": "#/definitions/positiveIntegerDefault0" },
"pattern": {
"type": "string",
"format": "regex"
},
"additionalItems": {
"anyOf": [
{ "type": "boolean" },
{ "$ref": "#" }
],
"default": {}
},
"items": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/schemaArray" }
],
"default": {}
},
"maxItems": { "$ref": "#/definitions/positiveInteger" },
"minItems": { "$ref": "#/definitions/positiveIntegerDefault0" },
"uniqueItems": {
"type": "boolean",
"default": false
},
"maxProperties": { "$ref": "#/definitions/positiveInteger" },
"minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" },
"required": { "$ref": "#/definitions/stringArray" },
"additionalProperties": {
"anyOf": [
{ "type": "boolean" },
{ "$ref": "#" }
],
"default": {}
},
"definitions": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"properties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"patternProperties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"dependencies": {
"type": "object",
"additionalProperties": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/stringArray" }
]
}
},
"enum": {
"type": "array",
"minItems": 1,
"uniqueItems": true
},
"type": {
"anyOf": [
{ "$ref": "#/definitions/simpleTypes" },
{
"type": "array",
"items": { "$ref": "#/definitions/simpleTypes" },
"minItems": 1,
"uniqueItems": true
}
]
},
"allOf": { "$ref": "#/definitions/schemaArray" },
"anyOf": { "$ref": "#/definitions/schemaArray" },
"oneOf": { "$ref": "#/definitions/schemaArray" },
"not": { "$ref": "#" }
"type": "object",
"additionalProperties": {
"$ref": "#"
},
"default": {}
},
"patternProperties": {
"type": "object",
"additionalProperties": {
"$ref": "#"
},
"default": {}
},
"dependencies": {
"exclusiveMaximum": [ "maximum" ],
"exclusiveMinimum": [ "minimum" ]
"type": "object",
"additionalProperties": {
"anyOf": [
{
"$ref": "#"
},
{
"$ref": "#/definitions/stringArray"
}
]
}
},
"enum": {
"type": "array",
"minItems": 1,
"uniqueItems": true
},
"type": {
"anyOf": [
{
"$ref": "#/definitions/simpleTypes"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/simpleTypes"
},
"minItems": 1,
"uniqueItems": true
}
]
},
"allOf": {
"$ref": "#/definitions/schemaArray"
},
"anyOf": {
"$ref": "#/definitions/schemaArray"
},
"oneOf": {
"$ref": "#/definitions/schemaArray"
},
"default": {}
"not": {
"$ref": "#"
}
},
"dependencies": {
"exclusiveMaximum": [
"maximum"
],
"exclusiveMinimum": [
"minimum"
]
},
"default": {}
}
import mock
import unittest
from collections import defaultdict
from slapos.recipe import request
from testfixtures import LogCapture
class RecipeTestMixin(object):
def setUp(self):
self.buildout = {
"buildout": {
},
"slap-connection": {
}
}
slap_patch = mock.patch(
"slapos.recipe.request.slapmodule.slap", autospec=True)
slap = slap_patch.start()
self.addCleanup(slap_patch.stop)
slap_instance = mock.MagicMock()
self.request_instance = mock.MagicMock()
register_instance = mock.MagicMock()
requested_instance = mock.MagicMock()
self.request_instance.return_value = requested_instance
register_instance.request = self.request_instance
slap_instance.registerComputerPartition.return_value = register_instance
slap.return_value = slap_instance
self.instance_getConnectionParameter = \
requested_instance.getConnectionParameter
def test_no_return_in_options_logs(self):
options = defaultdict(str)
self.instance_getConnectionParameter.return_value = self.return_value_empty
with LogCapture() as log:
self.recipe(self.buildout, "request", options)
log.check(
('request', 'DEBUG',
'No parameter to return to main instance.Be careful about that...'),
)
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
def test_return_in_options_logs(self):
options = defaultdict(str)
options['return'] = 'anything'
self.instance_getConnectionParameter.return_value = self.return_value_empty
with LogCapture() as log:
self.recipe(self.buildout, "request", options)
log.check()
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
def test_return_not_ready(self):
options = defaultdict(str)
options['return'] = 'anything'
self.instance_getConnectionParameter.side_effect = \
request.slapmodule.NotFoundError()
recipe = self.recipe(self.buildout, "request", options)
if self.raises:
self.assertRaises(KeyError, recipe.install)
self.assertEqual(options['connection-anything'], '')
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
def test_return_ready(self):
options = defaultdict(str)
options['return'] = 'anything'
self.instance_getConnectionParameter.return_value = self.return_value
recipe = self.recipe(self.buildout, "request", options)
result = recipe.install()
self.assertEqual([], result)
self.assertEqual(options['connection-anything'], 'done')
self.request_instance.assert_called_with(
'', 'RootSoftwareInstance', '', filter_kw={},
partition_parameter_kw=self.called_partition_parameter_kw,
shared=False, state='started')
class RecipeTest(RecipeTestMixin, unittest.TestCase):
recipe = request.Recipe
raises = True
return_value_empty = {}
return_value = 'done'
called_partition_parameter_kw = {}
class RequestOptionalTest(RecipeTestMixin, unittest.TestCase):
recipe = request.RequestOptional
raises = False
return_value = 'done'
return_value_empty = {}
called_partition_parameter_kw = {}
class RequestJSONEncodedTest(RecipeTestMixin, unittest.TestCase):
recipe = request.RequestJSONEncoded
return_value_empty = "{}"
return_value = '{"anything": "done"}'
raises = True
called_partition_parameter_kw = {'_': '{}'}
class RequestOptionalJSONEncodedTest(RecipeTestMixin, unittest.TestCase):
recipe = request.RequestOptionalJSONEncoded
return_value_empty = "{}"
return_value = '{"anything": "done"}'
raises = False
called_partition_parameter_kw = {'_': '{}'}
......@@ -14,7 +14,10 @@
"serialisation": {
"description": "How the parameters and results are serialised",
"require": true,
"enum": ["xml", "json-in-xml"],
"enum": [
"xml",
"json-in-xml"
],
"type": "string"
},
"software-type": {
......@@ -35,7 +38,10 @@
},
"serialisation": {
"description": "How the parameters and results are serialised, if different from global setting",
"enum": ["xml", "json-in-xml"],
"enum": [
"xml",
"json-in-xml"
],
"type": "string"
},
"request": {
......@@ -48,11 +54,11 @@
"description": "URL, relative to Software Release base path, of a json schema for values published by instance of current software type",
"type": "string"
},
"software-type" : {
"software-type": {
"description": "Value to be used as software type instead of the software type id (in order to use multiple diferent forms for the same software type).",
"type": "string"
},
"shared" : {
"shared": {
"description": "Define if the request will request a Slave or Software Instance.",
"type": "boolean"
},
......@@ -69,4 +75,3 @@
},
"type": "object"
}
......@@ -29,6 +29,7 @@ import unittest
import os
import glob
import json
import collections
import slapos.test
import jsonschema
......@@ -41,20 +42,39 @@ def getSchemaValidator(filename):
json_file.close()
return json_dict
def createTest(path, json_dict):
def createValidatorTest(path, json_dict):
# Test that json is valid
def run(self, *args, **kwargs):
with open(path, "r") as json_file:
self.assertEquals(jsonschema.validate(json.loads(json_file.read()), json_dict), None)
json_file.close()
self.assertEqual(jsonschema.validate(json.load(json_file), json_dict), None)
return run
def createFormatTest(path, json_dict):
# Test that json match our formatting rules
def run(self, *args, **kwargs):
with open(path, "r") as json_file:
content = json_file.read()
# this is the format produced by `format-json` tool at the
# root of this repository.
# XXX it would be better to reuse the code.
self.assertEqual(
(json.dumps(
json.loads(content, object_pairs_hook=collections.OrderedDict),
sort_keys=False,
indent=2,
separators=(',', ': ')) + "\n").splitlines(),
content.splitlines())
return run
def generateSoftwareCfgTest():
json_dict = getSchemaValidator("schema.json")
base_path = "/".join(slapos.test.__file__.split("/")[:-3])
for path in glob.glob("%s/software/*/software.cfg.json" % base_path):
test_name = "test_%s_software_cfg_json" % path.split("/")[-2]
setattr(TestJSONSchemaValidation, test_name , createTest(path, json_dict))
setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
def generateJSONSchemaTest():
......@@ -64,7 +84,8 @@ def generateJSONSchemaTest():
software_type = path.split("/")[-2]
filename = path.split("/")[-1].replace("-", "_").replace(".", "_")
test_name = "test_schema_%s_%s" % (software_type, filename)
setattr(TestJSONSchemaValidation, test_name , createTest(path, json_dict))
setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
class TestJSONSchemaValidation(unittest.TestCase):
pass
......
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"public-ipv4": {
"title": "Public IPv4",
"description": "Public ipv4 of the frontend (the one Apache will be indirectly listening to).",
"type": "string"
},
"ip-read-limit": {
"title": "IPReadLimit",
"description": "Value used to set IPReadLimit Parameter for antiloris.",
"type": "integer",
"default": 10
},
"mpm-server-limit": {
"title": "ServerLimit",
"description": "Value used to set ServerLimit on apache configuration.",
"type": "integer",
"default": 16
},
"mpm-max-clients": {
"title": "MaxClients",
"description": "Value used to set MaxClients on apache configuration.",
"type": "integer",
"default": 400
},
"mpm-start-servers": {
"title": "StartServers",
"description": "Value used to set StartServers on apache configuration.",
"type": "integer",
"default": 3
},
"mpm-thread-per-child": {
"title": "ThreadsPerChild",
"description": "Value used to set ThreadsPerChild on apache configuration.",
"type": "integer",
"default": 25
},