Skip to content

slapos
slapos
Commit 3b9c0e5d authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

re6st-registry: Refactor integration with re6st registry 

  * Follow up methods changes on re6stnet registry API.
  * getIPv4Information was simplifed on registry, so reimplement it.
  * Remove scripts generated and keep a single script to orchestate the token management.
  * Drop some code duplication
  * Remove babeld, openvpn and miniupnp as they are not required.
  * Remove useless scripts.
  * Remove re6stnet section, extra-eggs is more them enough and prevent duplications.
parent 6902a67f
Hide whitespace changes
Inline Side-by-side
Showing with 92 additions and 259 deletions
slapos/recipe/re6stnet/__init__.py
View file @ 3b9c0e5d
...@@ -218,24 +218,6 @@ class Recipe(GenericBaseRecipe): ...@@ -218,24 +218,6 @@ class Recipe(GenericBaseRecipe):
) )
path_list.append(request_add) path_list.append(request_add)
request_drop = self.createPythonScript(
self.options['drop-service-wrapper'].strip(),
'%s.re6stnet.requestRemoveToken' % __name__, service_dict
)
path_list.append(request_drop)
request_check = self.createPythonScript(
self.options['check-service-wrapper'].strip(),
'%s.re6stnet.checkService' % __name__, service_dict
)
path_list.append(request_check)
revoke_check = self.createPythonScript(
self.options['revoke-service-wrapper'].strip(),
'%s.re6stnet.requestRevoqueCertificate' % __name__, service_dict
)
path_list.append(revoke_check)
# Send connection parameters of slave instances # Send connection parameters of slave instances
if token_dict: if token_dict:
self.slap.initializeConnection(self.server_url, self.key_file, self.slap.initializeConnection(self.server_url, self.key_file,
......
slapos/recipe/re6stnet/re6stnet.py
View file @ 3b9c0e5d
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import httplib
import logging import logging
import json import json
import os import os
import time import time
import sqlite3
import slapos import slapos
import traceback import traceback
import logging import logging
import socket from re6st import registry
import select
from re6st import tunnel, ctl, registry, utils, x509
from OpenSSL import crypto
log = logging.getLogger('SLAPOS-RE6STNET') log = logging.getLogger('SLAPOS-RE6STNET')
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
logging.trace = logging.debug logging.trace = logging.debug
class iterRoutes(object):
_waiting = True
def __new__(cls, control_socket, network):
self = object.__new__(cls)
c = ctl.Babel(control_socket, self, network)
c.request_dump()
while self._waiting:
args = {}, {}, ()
c.select(*args)
utils.select(*args)
return (prefix
for neigh_routes in c.neighbours.itervalues()
for prefix in neigh_routes[1]
if prefix)
def babel_dump(self):
self._waiting = False
def loadJsonFile(path): def loadJsonFile(path):
if os.path.exists(path): if os.path.exists(path):
...@@ -57,12 +34,11 @@ def readFile(path): ...@@ -57,12 +34,11 @@ def readFile(path):
return content return content
return '' return ''
def getDb(db_path): def updateFile(file_path, value):
db = sqlite3.connect(db_path, isolation_level=None, if readFile(file_path) != value:
check_same_thread=False) writeFile(file_path, value)
db.text_factory = str return True
return False
return db.cursor()
def bang(args): def bang(args):
computer_guid = args['computer_id'] computer_guid = args['computer_id']
...@@ -77,64 +53,62 @@ def bang(args): ...@@ -77,64 +53,62 @@ def bang(args):
partition.bang(message='Published parameters changed!') partition.bang(message='Published parameters changed!')
log.info("Bang with message 'parameters changed'...") log.info("Bang with message 'parameters changed'...")
def requestAddToken(client, base_token_path):
def requestAddToken(args, can_bang=True):
time.sleep(3) time.sleep(3)
registry_url = args['registry_url']
base_token_path = args['token_base_path']
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')] path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
updated = False
log.info("Searching tokens to add at %s and found %s." % (base_token_path, path_list))
if not path_list: if not path_list:
log.info("No new token to add. Exiting...") log.info("No new token to add. Exiting...")
return return
client = registry.RegistryClient(registry_url)
call_bang = False
for reference_key in path_list: for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key) request_file = os.path.join(base_token_path, reference_key)
token = readFile(request_file) token = readFile(request_file)
log.info("Including token %s for %s" % (token, reference_key))
if token : if token :
reference = reference_key.split('.')[0] reference = reference_key.split('.')[0]
# email is unique as reference is also unique # email is unique as reference is also unique
email = '%s@slapos' % reference.lower() email = '%s@slapos' % reference.lower()
try: try:
result = client.requestAddToken(token, email) result = client.addToken(email, token)
except Exception: except Exception:
log.debug('Request add token fail for %s... \n %s' % (request_file, log.info('Request add token fail for %s... \n %s' % (request_file,
traceback.format_exc())) traceback.format_exc()))
continue continue
if result and result == token: if result and result == token:
# update information # update information
log.info("New token added for slave instance %s. Updating file status..." % log.info("New token added for slave instance %s. Updating file status..." %
reference) reference)
writeFile(os.path.join(base_token_path, '%s.status' % reference), status_file = os.path.join(base_token_path, '%s.status' % reference)
'TOKEN_ADDED') updateFile(status_file, 'TOKEN_ADDED')
os.unlink(request_file) os.unlink(request_file)
call_bang = True updated = True
else: else:
log.debug('Bad token. Request add token fail for %s...' % request_file) log.debug('Bad token. Request add token fail for %s...' % request_file)
if can_bang and call_bang: return updated
bang(args)
def requestRemoveToken(args): def requestRemoveToken(client, base_token_path):
base_token_path = args['token_base_path']
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')] path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
if not path_list: if not path_list:
log.info("No token to delete. Exiting...") log.info("No token to delete. Exiting...")
return return
client = registry.RegistryClient(args['registry_url'])
for reference_key in path_list: for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key) request_file = os.path.join(base_token_path, reference_key)
token = readFile(request_file) token = readFile(request_file)
if token : if token :
reference = reference_key.split('.')[0] reference = reference_key.split('.')[0]
try: try:
result = client.requestDeleteToken(token) result = client.deleteToken(token)
except httplib.NOTFOUND:
# Token is alread removed.
result = True
except Exception: except Exception:
log.debug('Request delete token fail for %s... \n %s' % (request_file, log.debug('Request delete token fail for %s... \n %s' % (request_file,
traceback.format_exc())) traceback.format_exc()))
...@@ -142,10 +116,12 @@ def requestRemoveToken(args): ...@@ -142,10 +116,12 @@ def requestRemoveToken(args):
else: else:
# certificate is invalidated, it will be revoked # certificate is invalidated, it will be revoked
writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '') writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
if result == 'True':
if result in (True, 'True'):
# update information # update information
log.info("Token deleted for slave instance %s. Clean up file status..." % log.info("Token deleted for slave instance %s. Clean up file status..." %
reference) reference)
if result in ['True', 'False']: if result in ['True', 'False']:
os.unlink(request_file) os.unlink(request_file)
status_file = os.path.join(base_token_path, '%s.status' % reference) status_file = os.path.join(base_token_path, '%s.status' % reference)
...@@ -159,198 +135,96 @@ def requestRemoveToken(args): ...@@ -159,198 +135,96 @@ def requestRemoveToken(args):
log.debug('Bad token. Request add token fail for %s...' % request_file) log.debug('Bad token. Request add token fail for %s...' % request_file)
def requestRevoqueCertificate(args): def requestRevoqueCertificate(args):
base_token_path = args['token_base_path'] base_token_path = args['token_base_path']
db = getDb(args['db'])
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')] path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
client = registry.RegistryClient(args['registry_url'])
for reference_key in path_list: for reference_key in path_list:
reference = reference_key.split('.')[0] reference = reference_key.split('.')[0]
# XXX - email is always unique
email = '%s@slapos' % reference.lower() if revokeByMail(args['registry_url'],
cert_string = '' '%s@slapos' % reference.lower(),
try: args['db']):
cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
(email,)).next()
except StopIteration:
# Certificate was not generated yet !!!
pass
try:
if cert_string:
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
cn = x509.subnetFromCert(cert)
result = client.revoke(str(cn))
time.sleep(2)
except Exception:
log.debug('Request revoke certificate fail for %s... \n %s' % (reference,
traceback.format_exc()))
continue
else:
os.unlink(os.path.join(base_token_path, reference_key)) os.unlink(os.path.join(base_token_path, reference_key))
log.info("Certificate revoked for slave instance %s." % reference) log.info("Certificate revoked for slave instance %s." % reference)
return
log.info("Failed to revoke email for %s" % reference)
def dumpIPv6Network(slave_reference, db, network, ipv6_file): def checkService(client, base_token_path, token_json):
email = '%s@slapos' % slave_reference.lower() token_dict = loadJsonFile(token_json)
updated = False
try:
cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
(email,)).next()
except StopIteration:
# Certificate was not generated yet !!!
pass
try:
if cert_string:
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
cn = x509.subnetFromCert(cert)
subnet = network + utils.binFromSubnet(cn)
ipv6 = utils.ipFromBin(subnet)
changed = readFile(ipv6_file) != ipv6
writeFile(ipv6_file, ipv6)
return ipv6, utils.binFromSubnet(cn), changed
except Exception:
log.debug('XXX for %s... \n %s' % (slave_reference,
traceback.format_exc()))
def sendto(sock, prefix, code):
return sock.sendto("%s\0%c" % (prefix, code), ('::1', tunnel.PORT))
def recv(sock, code):
try:
prefix, msg = sock.recv(1<<16).split('\0', 1)
int(prefix, 2)
except ValueError:
pass
else:
if msg and ord(msg[0]) == code:
return prefix, msg[1:]
return None, None
def dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list):
try:
if ipv6_prefix == "00000000000000000000000000000000":
# workarround to ignore the first node
ipv4 = "0.0.0.0"
changed = readFile(ipv4_file) != ipv4
writeFile(ipv4_file, ipv4)
return ipv4, changed
peers = []
peer_list = [prefix for prefix in peer_prefix_list if prefix == ipv6_prefix ]
if len(peer_list) == 0:
raise ValueError("Unable to find such prefix on database")
peer = peer_list[0]
sendto(sock, peer, 1)
s = sock,
timeout = 15
end = timeout + time.time()
while select.select(s, (), (), timeout)[0]:
prefix, msg = recv(sock, 1)
if prefix == peer:
break
timeout = max(0, end - time.time())
else:
logging.info("Timeout while querying address for %s/%s", int(peer, 2), len(peer))
msg = ""
if "," in msg:
ipv4 = msg.split(',')[0]
else:
ipv4 = "0.0.0.0"
changed = readFile(ipv4_file) != ipv4
writeFile(ipv4_file, ipv4)
return ipv4, changed
except Exception:
log.info('XXX for %s... \n %s' % (ipv6_prefix,
traceback.format_exc()))
return "0.0.0.0", False
def checkService(args, can_bang=True):
base_token_path = args['token_base_path']
token_dict = loadJsonFile(args['token_json'])
if not token_dict: if not token_dict:
return return
db = getDb(args['db'])
call_bang = False
computer_guid = args['computer_id']
partition_id = args['partition_id']
slap = slapos.slap.slap()
client = registry.RegistryClient(args['registry_url'])
ca = client.getCa()
network = x509.networkFromCa(crypto.load_certificate(crypto.FILETYPE_PEM, ca))
sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
peer_prefix_list = [prefix for prefix in
iterRoutes("/var/run/re6stnet/babeld.sock", network)]
# Check token status # Check token status
for slave_reference, token in token_dict.iteritems(): for slave_reference, token in token_dict.iteritems():
log.info("%s %s" % (slave_reference, token))
status_file = os.path.join(base_token_path, '%s.status' % slave_reference) status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
ipv6_file = os.path.join(base_token_path, '%s.ipv6' % slave_reference)
ipv4_file = os.path.join(base_token_path, '%s.ipv4' % slave_reference)
if not os.path.exists(status_file): if not os.path.exists(status_file):
# This token is not added yet! # This token is not added yet!
log.info("Token %s dont exist yet." % status_file) log.info("Token %s dont exist yet." % status_file)
continue continue
if not client.isToken(str(token)):
# Token is used to register client
updated = True
updateFile(status_file, 'TOKEN_USED')
log.info("Token status of %s updated to 'used'." % slave_reference)
msg = readFile(status_file) msg = readFile(status_file)
log.info("Token %s has %s State." % (status_file, msg)) log.info("Token %s has %s State." % (status_file, msg))
if msg == 'TOKEN_USED':
log.info("Dumping ipv6...")
ipv6, ipv6_prefix, ipv6_changed = dumpIPv6Network(slave_reference, db, network, ipv6_file)
log.info("%s, IPV6 = %s, IPV6_PREFIX = %s" % (slave_reference, ipv6, ipv6_prefix))
_, ipv4_changed = dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list)
if ipv4_changed or ipv6_changed:
call_bang = True
continue
# Check if token is not in the database if msg == 'TOKEN_USED':
status = False
try:
token_found, = db.execute("SELECT token FROM token WHERE token = ?",
(token,)).next()
if token_found == token:
status = True
except StopIteration:
pass
if not status:
# Token is used to register client
call_bang = True
try: try:
writeFile(status_file, 'TOKEN_USED') log.info("Dumping ipv6...")
dumpIPv6Network(slave_reference, db, network, ipv6_file) email = '%s@slapos' % slave_reference.lower()
dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list) try:
log.info("Token status of %s updated to 'used'." % slave_reference) ipv6 = client.getIPv6Address(str(email))
ipv6_file = os.path.join(base_token_path, '%s.ipv6' % slave_reference)
ipv6_changed = updateFile(ipv6_file, ipv6)
except Exception:
log.info('Error for dump ipv6 for %s... \n %s' % (slave_reference,
traceback.format_exc()))
continue
log.info("%s, IPV6 = %s" % (slave_reference, ipv6))
log.info("Dumping ipv4...")
try:
ipv4 = client.getIPv4Information(str(email)) or "0.0.0.0"
ipv4_file = os.path.join(base_token_path, '%s.ipv4' % slave_reference)
ipv4_changed = updateFile(ipv4_file, ipv4)
except Exception:
log.info('Error for dump ipv4 for %s... \n %s' % (slave_reference,
traceback.format_exc()))
continue
log.info("%s, IPV4 = %s" % (slave_reference, ipv4))
except IOError: except IOError:
# XXX- this file should always exists log.debug('Error when writing in file %s. Could not update status of %s...' %
log.debug('Error when writing in file %s. Clould not update status of %s...' % (status_file, slave_reference))
(status_file, slave_reference))
if call_bang and can_bang: if not updated or ipv4_changed or ipv6_changed:
bang(args) updated = True
return updated
def manage(args, can_bang=True):
client = registry.RegistryClient(args['registry_url'])
base_token_path = args['token_base_path']
token_json = args['token_json']
def manage(args):
# Request Add new tokens # Request Add new tokens
requestAddToken(args) has_new_token = requestAddToken(client, base_token_path)
# Request delete removed token # Request delete removed token
requestRemoveToken(args) requestRemoveToken(client, base_token_path)
# check status of all token # check status of all token
checkService(args) changed = checkService(client, base_token_path, token_json)
if (has_new_token or changed) and can_bang:
bang(args)
software/re6stnet/instance-re6stnet.cfg.in
View file @ 3b9c0e5d
...@@ -138,9 +138,6 @@ command = {{ re6st_registry }} ...@@ -138,9 +138,6 @@ command = {{ re6st_registry }}
wrapper = ${directory:services}/re6st-registry wrapper = ${directory:services}/re6st-registry
pid-file = ${directory:run}/registry.pid pid-file = ${directory:run}/registry.pid
manager-wrapper = ${directory:bin}/re6stManageToken manager-wrapper = ${directory:bin}/re6stManageToken
check-service-wrapper = ${directory:bin}/re6stCheckService
drop-service-wrapper = ${directory:bin}/re6stManageDeleteToken
revoke-service-wrapper = ${directory:bin}/re6stRevokeCertificate
openssl-bin = {{ openssl_bin }}/openssl openssl-bin = {{ openssl_bin }}/openssl
python-bin = {{ python_bin }} python-bin = {{ python_bin }}
ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }} ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }}
...@@ -167,26 +164,12 @@ recipe = slapos.cookbook:wrapper ...@@ -167,26 +164,12 @@ recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:script}/re6st-token-manager wrapper-path = ${directory:script}/re6st-token-manager
command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper} command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
[cron-entry-re6st-check] [cron-entry-re6st-manage]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = re6stnet-check-token name = re6stnet-check-token
frequency = */5 * * * * frequency = */5 * * * *
command = {{ python_bin }} ${re6st-registry:check-service-wrapper} command = {{ python_bin }} ${re6st-registry:manager-wrapper}
[cron-entry-re6st-revoke]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-revoke-cert
frequency = */5 * * * *
command = {{ python_bin }} ${re6st-registry:revoke-service-wrapper}
[cron-entry-re6st-drop]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-drop-token
frequency = */5 * * * *
command = {{ python_bin }} ${re6st-registry:drop-service-wrapper}
[logrotate-entry-re6stnet] [logrotate-entry-re6stnet]
< = logrotate-entry-base < = logrotate-entry-base
...@@ -227,9 +210,7 @@ parts = ...@@ -227,9 +210,7 @@ parts =
logrotate-entry-re6stnet logrotate-entry-re6stnet
re6stnet-manage re6stnet-manage
cron-entry-logrotate cron-entry-logrotate
cron-entry-re6st-check cron-entry-re6st-manage
cron-entry-re6st-drop
cron-entry-re6st-revoke
apache-httpd apache-httpd
apache-httpd-graceful apache-httpd-graceful
publish publish
......
software/re6stnet/software.cfg
View file @ 3b9c0e5d
[buildout] [buildout]
extends = extends =
../../component/re6stnet/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
...@@ -23,8 +22,6 @@ parts += ...@@ -23,8 +22,6 @@ parts +=
slapos-cookbook slapos-cookbook
eggs eggs
dash dash
babeld
re6stnet
template template
[eggs] [eggs]
...@@ -45,7 +42,6 @@ eggs = ...@@ -45,7 +42,6 @@ eggs =
${python-cffi:egg} ${python-cffi:egg}
${python-cryptography:egg} ${python-cryptography:egg}
pyOpenSSL pyOpenSSL
miniupnpc
re6stnet re6stnet
[download-base] [download-base]
...@@ -87,7 +83,7 @@ extra-context = ...@@ -87,7 +83,7 @@ extra-context =
[template-re6stnet] [template-re6stnet]
< = download-base < = download-base
filename = instance-re6stnet.cfg.in filename = instance-re6stnet.cfg.in
md5sum = 6e9452d283e82e2f512a9f9edb17fe3a md5sum = 4596d91cef4184b97d257a68478b6330
[template-apache-conf] [template-apache-conf]
< = download-base < = download-base
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7