[slapconfiguration] Recipe reads partitions resource file (tests included)

README.rst

@@ -2,3 +2,14 @@ slapos.cookbook
 ===============
 
 Cookbook of SlapOS recipes.
+
+
+testing
+=======
+
+Unit tests for recipes can be found under ``slapos/test/recipe``. To run the
+tests use provided unittest.defaultTestLoader inside ``slapos/test/test_recipe``
+by invoking 
+
+    python setup.py test --test-suite slapos.test.test_recipe.additional_tests
+

slapos/recipe/slapconfiguration.py

@@ -26,6 +26,7 @@
 ##############################################################################
 
 import json
+import logging
 import os
 
 import slapos.slap
@@ -33,15 +34,21 @@ from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
 from slapos.util import mkdir_p
+from slapos import format as slapformat
+
+
+logger = logging.getLogger("slapos")
+
 
 class Recipe(object):
   """
-  Retrieves slap partition parameters, and makes them available to other
-  buildout section in various ways, and in various encodings.
-  Populates the buildout section it is used in with all slap partition
-  parameters.
-  Also provides access to partition properties: all IPv4, IPv6 and tap
-  interfaces it is allowed to use.
+  Retrieve slap partition parameters and make them available in buildout section.
+
+  There are two sources of parameters. First is configuration file slapos.cfg and
+  derived information.
+  Second is partition's resource_file which is made available in form of keys joined
+  with "-" and with all "_" replaced by "-".
+  For example {"tun": {"ipv4": <addr>}} would be available in buildout as ${instance:tun-ipv4}.
 
   Input:
     url
@@ -67,7 +74,7 @@ class Recipe(object):
       Example:
         ${storage-configuration:storage-home}
 
-  Output:
+  Output (keys derived from SlapOS configuration):
     root-instance-title
       Hosting subscription or root instance title
     instance-title
@@ -122,6 +129,12 @@ class Recipe(object):
           options['configuration.' + key] = value
 
   def fetch_parameter_dict(self, options, instance_root):
+      """Gather parameters about current computer and partition.
+
+      Use two sources of truth
+      1. SlapOS Master - for external computer/partition information
+      2. format.Partition.resource_file - for partition specific details
+      """
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -233,6 +246,26 @@ class Recipe(object):
       options['storage-dict'] = storage_dict
 
       options['tap'] = tap_set
+
+      # The external information transfered from Slap Master has been processed
+      # so we extend with information gathered from partition resource file
+      resource_home = instance_root
+      while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
+        resource_home = os.path.normpath(os.path.join(resource_home, '..'))
+        if resource_home == "/":
+          break
+      else:
+        # no break happened - let's add partition resources into options
+        logger.debug("Using partition resource file {}".format(
+          os.path.join(resource_home, slapformat.Partition.resource_file)))
+        with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
+          partition_params = json.load(fi)
+        # be very careful with overriding master's information
+        for key, value in flatten_dict(partition_params).items():
+          if key not in options:
+            options[key] = value
+      # print out augmented options to see what we are passing
+      logger.debug(str(options))
       return self._expandParameterDict(options, parameter_dict)
 
   def _expandParameterDict(self, options, parameter_dict):
@@ -261,3 +294,14 @@ class JsonDump(Recipe):
 
     update = install
 
+
+def flatten_dict(data, key_prefix=''):
+  """Transform folded dict into one-level key-subkey-subsubkey dictionary."""
+  output = {}
+  for key, value in data.items():
+    prefixed_key = key_prefix + key.replace("_", "-")  # to be consistent with `fetch_parameter_dict`
+    if isinstance(value, dict):
+      output.update(flatten_dict(value, key_prefix=prefixed_key + "-"))
+      continue
+    output[prefixed_key] = value
+  return output

slapos/test/recipe/test_slaposconfiguration.py

+# coding: utf-8
+import json
+import mock
+import os
+import unittest
+from collections import defaultdict
+from slapos.recipe import slapconfiguration
+from slapos import format as slapformat
+
+
+class SlapConfigurationTest(unittest.TestCase):
+
+	def setUp(self):
+		"""Prepare files on filesystem."""
+		self.instance_root = "/tmp/instance_test_resourcefile"
+		os.mkdir(self.instance_root)
+		# create testing resource file
+		self.resource_file = os.path.join(self.instance_root, slapformat.Partition.resource_file)
+		self.resource = {
+			"tun": {
+				"ipv4": "192.168.0.1"
+			},
+			"address_list": [
+				10, 20
+			]
+		}
+		with open(self.resource_file, "wt") as fo:
+			json.dump(self.resource, fo)
+		# do your tests inside try block and clean up in finally
+		self.buildout = {
+			"buildout": {
+				"directory": self.instance_root
+			}
+		}
+
+	def tearDown(self):
+		os.unlink(self.resource_file)
+		os.rmdir(self.instance_root)
+
+	@mock.patch("slapos.slap.slap")
+	def test_correct_naming(self, MockClient):
+		"""Test correct naming of variables from resource file."""
+		MockClient.initializeConnection.return_value = None
+		MockClient.getInstanceParameterDict.return_value = dict()
+	
+		options = defaultdict(str)
+		recipe = slapconfiguration.Recipe(self.buildout, "slapconfiguration", options)
+
+		self.assertEqual(options['tun-ipv4'], "192.168.0.1",
+      "Folded attrs should be separated by -")
+
+		self.assertEqual(options['address-list'], [10, 20],
+      "All underscores should be replaced with -")

[Jérome Perrin]

@katomaso @rafael adding this empty slapos/test/recipe/__init__.py cause all the tests in this folder to run twice. I guess this was not intended, is it ?

[Rafael Monnerat]

No it is not intent.

[Jérome Perrin]

thanks, please check !234 (merged) then