proxy: support for software destruction

This introduces a new `state` column in `software` table.

Because node already has support for reporting various states of software
lifetime, this implementation stores the actual state of the software
installation.
What's not clear is that maybe it should store the requested state and the current state.

```plantuml
@startuml
start
repeat
  -> client request software in state available \n ""supplySupply?state="available""";
  if (already available ?) then (yes)
    :available;
  else (no )
    :install_requested;
    -> node starts building \n ""buildingSoftwareRelease"";
    :building;
    if ( build result? ) then ( node report build successful \n ""availableSoftwareRelease"" )
      :available;
    else (node report build error \n ""softwareReleaseError"" )
      :error;
    endif
  endif
repeat while ()

-> client request software in state destroyed \n ""supplySupply?state="destroyed""";;
:destroyed;
-> node notify software is deleted \n""destroyedSoftwareRelease"";
stop
@enduml
```

slapos/proxy/schema.sql

---version:11
+--version:12
 CREATE TABLE IF NOT EXISTS software%(version)s (
   url VARCHAR(255),
   computer_reference VARCHAR(255) DEFAULT '%(computer)s',
+  slap_state VARCHAR(255) DEFAULT 'available',
   CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
 );
 

slapos/proxy/views.py

@@ -225,8 +225,12 @@ def getFullComputerInformation():
   slap_computer = Computer(computer_id)
   slap_computer._software_release_list = []
   for sr in execute_db('software', 'select * from %s WHERE computer_reference=?', [computer_id]):
-    slap_computer._software_release_list.append(SoftwareRelease(
-      software_release=sr['url'], computer_guid=computer_id))
+    slap_computer._software_release_list.append(
+      SoftwareRelease(
+        software_release=sr['url'],
+        computer_guid=computer_id,
+        requested_state=sr['slap_state']
+      ))
   slap_computer._computer_partition_list = []
   for partition in execute_db('partition', 'SELECT * FROM %s WHERE computer_reference=?', [computer_id]):
     slap_computer._computer_partition_list.append(partitiondict2partition(
@@ -252,15 +256,35 @@ def setComputerPartitionConnectionXml():
 
 @app.route('/buildingSoftwareRelease', methods=['POST'])
 def buildingSoftwareRelease():
-  return 'Ignored'
+  execute_db(
+    'software',
+    'INSERT OR REPLACE INTO %s VALUES(?, ?, "building")',
+    [request.form['url'], request.form['computer_id']])
+  return 'OK'
+
+@app.route('/destroyedSoftwareRelease', methods=['POST'])
+def destroyedSoftwareRelease():
+  execute_db(
+    'software',
+    'DELETE FROM %s WHERE url = ? and computer_reference=? ',
+    [request.form['url'], request.form['computer_id']])
+  return 'OK'
 
 @app.route('/availableSoftwareRelease', methods=['POST'])
 def availableSoftwareRelease():
-  return 'Ignored'
+  execute_db(
+    'software',
+    'UPDATE %s SET slap_state="available" WHERE url=? AND computer_reference=?',
+    [request.form['url'], request.form['computer_id']])
+  return 'OK'
 
 @app.route('/softwareReleaseError', methods=['POST'])
 def softwareReleaseError():
-  return 'Ignored'
+  execute_db(
+    'software',
+    'INSERT OR REPLACE INTO %s VALUES(?, ?, "error")',
+    [request.form['url'], request.form['computer_id']])
+  return 'OK'
 
 @app.route('/softwareInstanceError', methods=['POST'])
 def softwareInstanceError():
@@ -319,11 +343,26 @@ def registerComputerPartition():
 def supplySupply():
   url = request.form['url']
   computer_id = request.form['computer_id']
-  if request.form['state'] == 'destroyed':
-    execute_db('software', 'DELETE FROM %s WHERE url = ? AND computer_reference=?',
-               [url, computer_id])
-  else:
-    execute_db('software', 'INSERT OR REPLACE INTO %s VALUES(?, ?)', [url, computer_id])
+  state = request.form['state']
+  if state not in ('available', 'destroyed'):
+    raise ValueError("Wrong state %s" % state)
+
+  if state == 'available':
+    software = execute_db(
+        'software',
+        'SELECT slap_state FROM %s WHERE url=? and computer_reference=?',
+        [url, computer_id], one=True)
+    if software and software['slap_state'] == 'available':
+      return '%r already available' % url
+    state = 'install_requested'
+
+  execute_db(
+    'software',
+    'INSERT OR REPLACE INTO %s VALUES(?, ?, ?)',
+    [url, computer_id, state])
+
+  if state == 'destroyed':
+    return '%r destroyed' % url
   return '%r added' % url
 
 

slapos/slap/slap.py

@@ -120,7 +120,7 @@ class SoftwareRelease(SlapDocument):
   Contains Software Release information
   """
 
-  def __init__(self, software_release=None, computer_guid=None, **kw):
+  def __init__(self, software_release=None, computer_guid=None, requested_state='available', **kw):
     """
     Makes easy initialisation of class parameters
 
@@ -131,9 +131,10 @@ class SoftwareRelease(SlapDocument):
     self._software_instance_list = []
     self._software_release = software_release
     self._computer_guid = computer_guid
+    self._requested_state = requested_state
 
   def __getinitargs__(self):
-    return (self._software_release, self._computer_guid, )
+    return (self._software_release, self._computer_guid, self._requested_state)
 
   def getComputerId(self):
     if not self._computer_guid:
@@ -173,7 +174,7 @@ class SoftwareRelease(SlapDocument):
       'computer_id': self.getComputerId()})
 
   def getState(self):
-    return getattr(self, '_requested_state', 'available')
+    return self._requested_state
 
 
 @implementer(interface.ISoftwareProductCollection)

slapos/tests/slapproxy/__init__.py

@@ -327,7 +327,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     computer_partition.__dict__.update(software_instance.__dict__)
     return computer_partition
 
-  def supply(self, url, computer_id=None, state=''):
+  def supply(self, url, computer_id=None, state='available'):
     if not computer_id:
       computer_id = self.computer_id
     request_dict = {'url':url, 'computer_id': computer_id, 'state':state}
@@ -347,12 +347,72 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     """
     Return computer information as stored in proxy for corresponding id
     """
-    rv = self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id)
-    computer = loads(rv.data)
+    computer = self.getFullComputerInformation()
     for instance in computer._computer_partition_list:
       if instance._partition_id == computer_partition_id:
         return instance
 
+  def getFullComputerInformation(self):
+    return loads(self.app.get('/getFullComputerInformation?computer_id=%s' % self.computer_id).data)
+
+
+class TestSoftwareInstallation(MasterMixin, unittest.TestCase):
+  def setUp(self):
+    super(TestSoftwareInstallation, self).setUp()
+    self.software_release_url = self.id()
+
+  def assertSoftwareState(self, state):
+    sr, = self.getFullComputerInformation()._software_release_list
+    self.assertEqual(sr.getState(), state)
+
+  def test_installation_success(self):
+    # install_requested -> building -> available
+    self.supply(self.software_release_url)
+    self.assertSoftwareState('install_requested')
+
+    self.app.post('/buildingSoftwareRelease', data={
+      'url': self.software_release_url,
+      'computer_id': self.computer_id
+    })
+    self.assertSoftwareState('building')
+
+    self.app.post('/availableSoftwareRelease', data={
+      'url': self.software_release_url,
+      'computer_id': self.computer_id
+    })
+    self.assertSoftwareState('available')
+
+  def test_installation_failed(self):
+    # install_requested -> building -> error
+    self.supply(self.software_release_url)
+    self.assertSoftwareState('install_requested')
+
+    self.app.post('/buildingSoftwareRelease', data={
+      'url': self.software_release_url,
+      'computer_id': self.computer_id
+    })
+    self.assertSoftwareState('building')
+
+    self.app.post('/softwareReleaseError', data={
+      'url': self.software_release_url,
+      'computer_id': self.computer_id
+    })
+    self.assertSoftwareState('error')
+
+  def test_destroyed(self):
+    # any -> destroyed -> not present
+    self.supply(self.software_release_url)
+    self.assertSoftwareState('install_requested')
+
+    self.supply(self.software_release_url, state="destroyed")
+    self.assertSoftwareState('destroyed')
+
+    self.app.post('/destroyedSoftwareRelease', data={
+      'url': self.software_release_url,
+      'computer_id': self.computer_id
+    })
+    self.assertEqual([], self.getFullComputerInformation()._software_release_list)
+
 
 class TestRequest(MasterMixin):
   """
@@ -702,6 +762,7 @@ class TestSlaveRequest(MasterMixin):
          shared=True, filter_kw=dict(instance_guid=partition._instance_guid))
     self.assertEqual(slave._partition_id, partition._partition_id)
 
+
 class TestMultiNodeSupport(MasterMixin):
   def test_multi_node_support_different_software_release_list(self):
     """
@@ -775,17 +836,33 @@ class TestMultiNodeSupport(MasterMixin):
     computer_0 = loads(self.app.get('/getFullComputerInformation?computer_id=COMP-0').data)
     computer_1 = loads(self.app.get('/getFullComputerInformation?computer_id=COMP-1').data)
 
-    self.assertEqual(len(computer_0._software_release_list), 0)
-    self.assertEqual(len(computer_1._software_release_list), 1)
+    # at this point, software is requested to be destroyed on COMP-0
+    self.assertEqual(
+        computer_0._software_release_list[0].getURI(),
+        software_release_url
+    )
+    self.assertEqual(
+        computer_0._software_release_list[0].getComputerId(),
+        'COMP-0'
+    )
+    self.assertEqual(
+        computer_0._software_release_list[0].getState(),
+        'destroyed'
+    )
 
+    # but is still requested for installation on COMP-1
     self.assertEqual(
-        computer_1._software_release_list[0]._software_release,
+        computer_1._software_release_list[0].getURI(),
         software_release_url
     )
     self.assertEqual(
-        computer_1._software_release_list[0]._computer_guid,
+        computer_1._software_release_list[0].getComputerId(),
         'COMP-1'
     )
+    self.assertEqual(
+        computer_1._software_release_list[0].getState(),
+        'install_requested'
+    )
 
   def test_multi_node_support_instance_default_computer(self):
     """
@@ -974,6 +1051,7 @@ class TestMultiNodeSupport(MasterMixin):
     except slapos.slap.NotFoundError:
       self.fail('Could not fetch informations for registered computer.')
 
+
 class TestMultiMasterSupport(MasterMixin):
   """
   Test multimaster support in slapproxy.
@@ -1262,25 +1340,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db
 
 # XXX: when testing new schema version,
 # rename to "TestMigrateVersion10ToLatest" and test accordingly.
-# Of course, also test version 11 to latest (should be 12).
-class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
+# Of course, also test version 12 to latest (should be 13).
+class TestMigrateVersion10To12(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
   """
   Test that old database version are automatically migrated without failure
   """
   def setUp(self):
-    super(TestMigrateVersion10To11, self).setUp()
+    super(TestMigrateVersion10To12, self).setUp()
     schema = bytes2str(pkg_resources.resource_string(
       'slapos.tests.slapproxy',
       'database_dump_version_10.sql'
-    )) % dict(version='11')
+    )) % dict(version='12')
     self.db = sqlite_connect(self.proxy_db)
     self.db.cursor().executescript(schema)
     self.db.commit()
 
   def test_automatic_migration(self):
-    table_list = ('software11', 'computer11', 'partition11', 'slave11', 'partition_network11')
+    table_list = ('software12', 'computer12', 'partition12', 'slave12', 'partition_network12')
     for table in table_list:
-      self.assertRaises(sqlite3.OperationalError, self.db.execute, "SELECT name FROM computer11")
+      self.assertRaises(sqlite3.OperationalError, self.db.execute, "SELECT name FROM computer12")
     # Run a dummy request to cause migration
     self.app.get('/getComputerInformation?computer_id=computer')
 
@@ -1291,31 +1369,31 @@ class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, T
     )
 
     # Lower level tests
-    computer_list = self.db.execute("SELECT * FROM computer11").fetchall()
+    computer_list = self.db.execute("SELECT * FROM computer12").fetchall()
     self.assertEqual(
         computer_list,
         [(u'computer', u'127.0.0.1', u'255.255.255.255')]
     )
 
-    software_list = self.db.execute("SELECT * FROM software11").fetchall()
+    software_list = self.db.execute("SELECT * FROM software12").fetchall()
     self.assertEqual(
         software_list,
-        [(u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'computer')]
+        [(u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'computer', u'available')]
     )
 
-    partition_list = self.db.execute("select * from partition11").fetchall()
+    partition_list = self.db.execute("select * from partition12").fetchall()
     self.assertEqual(
         partition_list,
         [(u'slappart0', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="json">{\n  "site-id": "erp5"\n  }\n}</parameter>\n</instance>\n', None, None, u'production', u'slapos', None, u'started'), (u'slappart1', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>\n</instance>\n', None, u'mariadb', u'MariaDB DataBase', u'slappart0', u'started'), (u'slappart2', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="cloudooo-json"></parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', None, u'cloudooo', u'Cloudooo', u'slappart0', u'started'), (u'slappart3', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">memcached://127.0.0.1:11000/</parameter>\n</instance>\n', None, u'memcached', u'Memcached', u'slappart0', u'started'), (u'slappart4', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">memcached://127.0.0.1:13301/</parameter>\n</instance>\n', None, u'kumofs', u'KumoFS', u'slappart0', u'started'), (u'slappart5', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>\n  <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>\n  <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">https://[fc00::1]:10001</parameter>\n</instance>\n', None, u'tidstorage', u'TidStorage', u'slappart0', u'started'), (u'slappart6', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart7', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart8', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart9', u'computer', u'free', None, None, None, None, None, None, None, u'started')]
     )
 
-    slave_list = self.db.execute("select * from slave11").fetchall()
+    slave_list = self.db.execute("select * from slave12").fetchall()
     self.assertEqual(
         slave_list,
         []
     )
 
-    partition_network_list = self.db.execute("select * from partition_network11").fetchall()
+    partition_network_list = self.db.execute("select * from partition_network12").fetchall()
     self.assertEqual(
         partition_network_list,
         [(u'slappart0', u'computer', u'slappart0', u'127.0.0.1', u'255.255.255.255'), (u'slappart0', u'computer', u'slappart0', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart1', u'computer', u'slappart1', u'127.0.0.1', u'255.255.255.255'), (u'slappart1', u'computer', u'slappart1', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart2', u'computer', u'slappart2', u'127.0.0.1', u'255.255.255.255'), (u'slappart2', u'computer', u'slappart2', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart3', u'computer', u'slappart3', u'127.0.0.1', u'255.255.255.255'), (u'slappart3', u'computer', u'slappart3', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart4', u'computer', u'slappart4', u'127.0.0.1', u'255.255.255.255'), (u'slappart4', u'computer', u'slappart4', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart5', u'computer', u'slappart5', u'127.0.0.1', u'255.255.255.255'), (u'slappart5', u'computer', u'slappart5', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart6', u'computer', u'slappart6', u'127.0.0.1', u'255.255.255.255'), (u'slappart6', u'computer', u'slappart6', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart7', u'computer', u'slappart7', u'127.0.0.1', u'255.255.255.255'), (u'slappart7', u'computer', u'slappart7', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart8', u'computer', u'slappart8', u'127.0.0.1', u'255.255.255.255'), (u'slappart8', u'computer', u'slappart8', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart9', u'computer', u'slappart9', u'127.0.0.1', u'255.255.255.255'), (u'slappart9', u'computer', u'slappart9', u'fc00::1', u'ffff:ffff:ffff::')]