Commit 7defe1aa authored by Julien Muchembled's avatar Julien Muchembled

slapproxy: skip instanciation if nothing has changed

parent 437b9e31
......@@ -43,6 +43,7 @@ import logging
import json
import shutil
import six
import errno
if sys.version_info < (2, 6):
warnings.warn('Used python version (%s) is old and has problems with'
......@@ -966,10 +967,7 @@ stderr_logfile_backups=1
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
parameter_dict = computer_partition.getInstanceParameterDict()
if 'timestamp' in parameter_dict:
timestamp = parameter_dict['timestamp']
else:
timestamp = None
timestamp = parameter_dict.get('timestamp')
error_output_file = os.path.join(
instance_path,
......@@ -1031,18 +1029,23 @@ stderr_logfile_backups=1
# If not: it's not worth processing this partition (nothing has
# changed).
if (computer_partition_id not in self.computer_partition_filter_list and
not self.develop and os.path.exists(timestamp_path)):
with open(timestamp_path) as f:
old_timestamp = f.read()
last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp:
try:
if periodicity == 0:
os.remove(timestamp_path)
elif int(timestamp) <= int(old_timestamp):
not self.develop and timestamp and periodicity):
try:
last_runtime = os.path.getmtime(timestamp_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
with open(timestamp_path) as f:
try:
old_timestamp = float(f.read())
except ValueError:
self.logger.exception('')
old_timestamp = 0
if float(timestamp) <= old_timestamp:
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
if time.time() <= last_runtime + periodicity or periodicity < 0:
# check promises anomaly
if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
self.logger.debug('Partition already up-to-date.')
......@@ -1055,13 +1058,7 @@ stderr_logfile_backups=1
manager.instanceTearDown(local_partition)
return
else:
# Periodicity forced processing this partition. Removing
# the timestamp file in case it fails.
os.remove(timestamp_path)
except ValueError:
os.remove(timestamp_path)
self.logger.exception('')
os.remove(timestamp_path)
# Include Partition Logging
log_folder_path = "%s/.slapgrid/log" % instance_path
......@@ -1168,7 +1165,7 @@ stderr_logfile_backups=1
# If partition has been successfully processed, write timestamp
if timestamp:
with open(timestamp_path, 'w') as f:
f.write(timestamp)
f.write(str(timestamp))
def FilterComputerPartitionList(self, computer_partition_list):
"""
......
--version:13
--version:14
CREATE TABLE IF NOT EXISTS software%(version)s (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s',
......@@ -26,6 +26,7 @@ CREATE TABLE IF NOT EXISTS partition%(version)s (
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
timestamp REAL,
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
......
......@@ -31,6 +31,7 @@
from lxml import etree
import random
import string
import time
from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError
......@@ -91,7 +92,8 @@ def partitiondict2partition(partition):
slap_partition._software_release_document = None
slap_partition._requested_state = 'destroyed'
slap_partition._need_modification = 0
slap_partition._instance_guid = '%s-%s' % (partition['computer_reference'], partition['reference'])
slap_partition._instance_guid = '%(computer_reference)s-%(reference)s' \
% partition
root_partition = getRootPartition(partition['reference'])
......@@ -118,6 +120,9 @@ def partitiondict2partition(partition):
loads(partition['slave_instance_list'].encode('utf-8'))
else:
slap_partition._parameter_dict['slave_instance_list'] = []
timestamp = partition['timestamp']
if timestamp:
slap_partition._parameter_dict['timestamp'] = str(timestamp)
slap_partition._connection_dict = xml2dict(partition['connection_xml'])
slap_partition._software_release_document = SoftwareRelease(
software_release=partition['software_release'],
......@@ -267,12 +272,18 @@ def setComputerPartitionConnectionXml():
query = 'UPDATE %s SET connection_xml=? WHERE reference=? AND computer_reference=?'
argument_list = [connection_xml, computer_partition_id, computer_id]
execute_db('partition', query, argument_list)
return 'done'
# Update timestamp of parent partition.
requested_by = execute_db('partition',
'SELECT requested_by FROM %s WHERE reference=? AND computer_reference=?',
(computer_partition_id, computer_id), one=True)['requested_by']
execute_db('partition',
"UPDATE %s SET timestamp=? WHERE reference=? AND computer_reference=?",
(time.time(), requested_by, computer_id))
else:
query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?'
argument_list = [connection_xml, computer_partition_id, slave_reference]
execute_db('slave', query, argument_list)
return 'done'
return 'done'
@app.route('/buildingSoftwareRelease', methods=['POST'])
def buildingSoftwareRelease():
......@@ -569,26 +580,23 @@ def getRootPartition(reference):
reference, execute_db("partition", "select reference, requested_by from %s")))
return None
parent_partition = execute_db('partition', p, [partition['requested_by']], one=True)
while (parent_partition is not None and
parent_partition['requested_by'] and
parent_partition['requested_by'] != reference):
while True:
requested_by = partition['requested_by']
if requested_by is None or requested_by == reference:
return partition
parent_partition = execute_db('partition', p, (requested_by,), one=True)
if parent_partition is None:
return partition
partition = parent_partition
reference = parent_partition['requested_by']
parent_partition = execute_db('partition', p, [reference], one=True)
return partition
reference = requested_by
def requestNotSlave(software_release, software_type, partition_reference, partition_id, partition_parameter_kw, filter_kw, requested_state):
instance_xml = dict2xml(partition_parameter_kw)
requested_computer_id = filter_kw['computer_guid']
args = []
a = args.append
q = 'SELECT * FROM %s WHERE partition_reference=?'
a(partition_reference)
partition = execute_db('partition', q, args, one=True)
partition = execute_db('partition',
'SELECT * FROM %s WHERE partition_reference=?',
(partition_reference,), one=True)
args = []
a = args.append
......@@ -601,8 +609,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
if partition is None:
app.logger.warning('No more free computer partition')
abort(404)
q += ' ,software_release=?'
a(software_release)
if partition_reference:
q += ' ,partition_reference=?'
a(partition_reference)
......@@ -612,10 +618,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
if not software_type:
software_type = 'RootSoftwareInstance'
else:
# XXX Check if software_release should be updated
if partition['software_release'] != software_release:
q += ' ,software_release=?'
a(software_release)
if partition['requested_by']:
root_partition = getRootPartition(partition['requested_by'])
if root_partition and root_partition['requested_state'] != "started":
......@@ -623,49 +625,49 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
# child can be stopped or destroyed while parent is started
requested_state = root_partition['requested_state']
if requested_state:
q += ', requested_state=?'
a(requested_state)
#
# XXX change software_type when requested
#
if software_type:
q += ' ,software_type=?'
a(software_type)
timestamp = partition['timestamp']
changed = timestamp is None
for k, v in (('requested_state', requested_state or
partition['requested_state']),
('software_release', software_release),
('software_type', software_type),
('xml', instance_xml)):
if partition[k] != v:
q += ', %s=?' % k
a(v)
changed = True
if changed:
timestamp = time.time()
q += ', timestamp=?'
a(timestamp)
# Else: only update partition parameters
if instance_xml:
q += ' ,xml=?'
a(instance_xml)
q += ' WHERE reference=? AND computer_reference=?'
a(partition['reference'])
a(partition['computer_reference'])
execute_db('partition', q, args)
args = []
partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
[partition['reference'], partition['computer_reference']], one=True)
address_list = []
for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]):
address_list.append((address['reference'], address['address']))
if not requested_state:
requested_state = 'started'
# XXX it should be ComputerPartition, not a SoftwareInstance
software_instance = SoftwareInstance(_connection_dict=xml2dict(partition['connection_xml']),
_parameter_dict=xml2dict(partition['xml']),
connection_xml=partition['connection_xml'],
slap_computer_id=partition['computer_reference'],
slap_computer_partition_id=partition['reference'],
slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url',
slap_software_type=partition['software_type'],
_instance_guid='%s-%s' % (partition['computer_reference'], partition['reference']),
_requested_state=requested_state,
ip_list=address_list)
return software_instance
parameter_dict = xml2dict(partition['xml'])
parameter_dict['timestamp'] = str(partition['timestamp'])
return SoftwareInstance(
_connection_dict=xml2dict(partition['connection_xml']),
_parameter_dict=parameter_dict,
connection_xml=partition['connection_xml'],
slap_computer_id=partition['computer_reference'],
slap_computer_partition_id=partition['reference'],
slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url',
slap_software_type=partition['software_type'],
_instance_guid='%(computer_reference)s-%(reference)s' % partition,
_requested_state=requested_state or 'started',
ip_list=address_list)
def requestSlave(software_release, software_type, partition_reference, partition_id, partition_parameter_kw, filter_kw, requested_state):
"""
......@@ -737,7 +739,6 @@ def requestSlave(software_release, software_type, partition_reference, partition
a(partition['reference'])
a(requested_computer_id)
execute_db('partition', q, args)
args = []
partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
[partition['reference'], requested_computer_id], one=True)
......@@ -758,16 +759,15 @@ def requestSlave(software_release, software_type, partition_reference, partition
address_list.append((address['reference'], address['address']))
# XXX it should be ComputerPartition, not a SoftwareInstance
software_instance = SoftwareInstance(_connection_dict=xml2dict(slave['connection_xml']),
_parameter_dict=xml2dict(instance_xml),
slap_computer_id=partition['computer_reference'],
slap_computer_partition_id=slave['hosted_by'],
slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url',
slap_software_type=partition['software_type'],
ip_list=address_list)
return software_instance
return SoftwareInstance(
_connection_dict=xml2dict(slave['connection_xml']),
_parameter_dict=xml2dict(instance_xml),
slap_computer_id=partition['computer_reference'],
slap_computer_partition_id=slave['hosted_by'],
slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url',
slap_software_type=partition['software_type'],
ip_list=address_list)
@app.route('/softwareInstanceRename', methods=['POST'])
def softwareInstanceRename():
......
......@@ -627,16 +627,16 @@ class ComputerPartition(SlapRequester):
def getInstanceParameter(self, key):
parameter_dict = getattr(self, '_parameter_dict', None) or {}
if key in parameter_dict:
try:
return parameter_dict[key]
else:
except KeyError:
raise NotFoundError("%s not found" % key)
def getConnectionParameter(self, key):
connection_dict = self.getConnectionParameterDict()
if key in connection_dict:
try:
return connection_dict[key]
else:
except KeyError:
raise NotFoundError("%s not found" % key)
def setUsage(self, usage_log):
......
This diff is collapsed.
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software13 (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
requested_state VARCHAR(255) DEFAULT 'available',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
INSERT INTO "software13" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
CREATE TABLE computer13 (
reference VARCHAR(255) DEFAULT 'computer',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
);
INSERT INTO "computer13" VALUES('computer','127.0.0.1','255.255.255.255');
CREATE TABLE partition13 (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255), -- name of the instance
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
INSERT INTO "partition13" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started');
INSERT INTO "partition13" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started');
INSERT INTO "partition13" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started');
INSERT INTO "partition13" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started');
INSERT INTO "partition13" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started');
INSERT INTO "partition13" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started');
INSERT INTO "partition13" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition13" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition13" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition13" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
CREATE TABLE slave13 (
reference VARCHAR(255), -- unique slave reference
computer_reference VARCHAR(255) DEFAULT 'computer',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network13 (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network13" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network13" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network13" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
CREATE TABLE forwarded_partition_request13 (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
COMMIT;
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment