[slapos.package] First working version for debian updates

  Still working in progress

setup.py

-from setuptools import setup
+from setuptools import setup, find_packages
 
-version = '0.0.1.1'
+# Still under development
+version = '0.0.1.3'
 name = 'slapos.package'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -16,7 +17,8 @@ setup(name=name,
       license='GPLv3',
       url='http://www.slapos.org',
       author='VIFIB',
-      packages=['slapos.package'],
+      namespace_packages=['slapos'],
+      packages=find_packages(),
       include_package_data=True,
       install_requires=[
           'slapos.libnetworkcache',
@@ -25,8 +27,16 @@ setup(name=name,
       zip_safe=False,
       entry_points={
           'console_scripts': [
-              'slapos-update = slapos.package.update:main',
-          ]
+              # Those entry points are development version
+              'slappkg-update = slapos.package.update:main',
+              'slappkg-discover = slapos.package.distribution:do_discover',
+              'slappkg-upload-key = slapos.package.upload_key:main'
+          ],
+
+        # Not supported yet
+        #'slapos.cli': [
+        #  'package upload-key = slapos.package.upload_key:main'
+        #  ]
       },
       test_suite="slapos.package.test",
 )

slapos/package/base_promise.py

@@ -55,7 +55,7 @@ class BasePromise(PackageManager):
     self.log("Calling: %s" % ' '.join(cmd_args))
 
     if not dry_run:
-      p = sub.Popen(cmd_args, stdout=stdout, stderr=stderr)
+      p = subprocess.Popen(cmd_args, stdout=stdout, stderr=stderr)
       output, err = p.communicate()
       return output, err
 

slapos/package/distribution.py

 import platform
+import glob
 import re
+import os
 
 _distributor_id_file_re = re.compile("(?:DISTRIB_ID\s*=)\s*(.*)", re.I)
 _release_file_re = re.compile("(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I)
@@ -30,6 +32,13 @@ def patched_linux_distribution(distname='', version='', id='',
     return platform.linux_distribution(distname, version, id, supported_dists, full_distribution_name)
 
 class PackageManager:
+
+  def matchSignatureList(self, signature_list):
+    return self.getOSSignature() in signature_list
+
+  def getOSSignature(self):
+    return "+++".join(patched_linux_distribution())
+
   def getDistributionName(self):
     return patched_linux_distribution()[0]
 
@@ -62,36 +71,60 @@ class PackageManager:
     """ Add a repository """
     return self._getDistribitionHandler().updateRepository(self._call)
 
-  def _installSoftware(self, name):
+  def _installSoftwareList(self, name_list):
     """ Upgrade softwares """
-    return self._getDistribitionHandler().installSoftware(self._call, name)
+    return self._getDistribitionHandler().installSoftwareList(self._call, name_list)
 
   def _updateSoftware(self):
     """ Upgrade softwares """
     return self._getDistribitionHandler().updateSoftware(self._call)
 
-  def updateSystem(self):
+  def _updateSystem(self):
     """ Dist-Upgrade of system """
     return self._getDistribitionHandler().updateSystem(self._call)
 
+  def update(self, repository_list=[], package_list=[]):
+    """ Perform upgrade """
+    self._purgeRepository()
+    for alias, url in repository_list:
+      self._addRepository(url, alias)
+    self._updateRepository()
+    if len(package_list):
+      self._installSoftwareList(package_list)
+
 # This helper implements API for package handling
 class AptGet:
+ 
+  source_list_path = "/etc/apt/sources.list"
+  source_list_d_path = "/etc/apt/sources.list.d"
+
   def purgeRepository(self, caller):
     """ Remove all repositories """
-    raise NotImplemented
+    # Aggressive removal
+    os.remove(self.source_list_path)
+    open("/etc/apt/sources.list", "w+").write("# Removed all")
+    for file_path in glob.glob("%s/*" % self.source_list_d_path):
+      os.remove(file_path)
 
   def addRepository(self, caller, url, alias):
     """ Add a repository """
-    raise NotImplemented
+    repos_file = open("%s/%s.list" % (self.source_list_d_path, alias), "w")
+    prefix = "deb "
+    if alias.endswith("-src"):
+      prefix = "deb-src "
+    repos_file.write(prefix + url)
+    repos_file.close()
 
   def updateRepository(self, caller):
     """ Add a repository """
     caller(['apt-get', 'update'], stdout=None)
 
-  def installSoftware(self, caller, name):
+  def installSoftwareList(self, caller, name_list):
     """ Instal Software """
     self.updateRepository(caller)
-    caller(["apt-get", "install", "-y", name], stdout=None) 
+    command_list = ["apt-get", "install", "-y"]
+    command_list.extend(name_list)
+    caller(command_list, stdout=None) 
 
   def isUpgradable(self, caller, name):
     output, err = caller(["apt-get", "upgrade", "--dry-run"])
@@ -132,10 +165,12 @@ class Zypper:
         return False
     return True
 
-  def installSoftware(self, caller, name):
+  def installSoftwareList(self, caller, name_list):
     """ Instal Software """
     self.updateRepository(caller)
-    caller(['zypper', '--gpg-auto-import-keys', 'up', '-ly', name], stdout=None) 
+    command_list = ['zypper', '--gpg-auto-import-keys', 'up', '-ly']
+    command_list.extend(name_list)
+    caller(command_list, stdout=None) 
 
   def updateSoftware(self, caller):
     """ Upgrade softwares """
@@ -145,3 +180,9 @@ class Zypper:
     """ Dist-Upgrade of system """
     caller(['zypper', '--gpg-auto-import-keys', 'dup', '-ly'], stdout=None)
 
+
+def do_discover():
+  package_manager = PackageManager()
+  print package_manager.getOSSignature()
+
+

slapos/package/signature.py

@@ -73,6 +73,14 @@ class NetworkCache:
     else:
       self.directory_key = "slapos-upgrade-testing-key"
 
+def get_yes_no(prompt):
+  while True:
+    answer = raw_input(prompt + " [y,n]: ")
+    if answer.upper() in ['Y', 'YES']:
+        return True
+    if answer.upper() in ['N', 'NO']:
+        return False
+
 class Signature:
 
   def __init__(self, config, logger=None):
@@ -92,6 +100,7 @@ class Signature:
       for entry in entry_list:
         if entry['timestamp'] > timestamp:
           best_entry = entry
+
       return best_entry
 
     return helper_download_network_cached_to_file(
@@ -145,7 +154,7 @@ class Signature:
     except Exception:
       print 'Unable to upload to cache:\n%s.' % traceback.format_exc()
 
-  def upload(self, dry_run=0):
+  def upload(self, dry_run=0, verbose=1):
     upgrade_info = ConfigParser.RawConfigParser()
     upgrade_info.read(self.config.upgrade_file)
 
@@ -160,13 +169,17 @@ class Signature:
     upgrade_info.write(file)
     file.close()
 
+    if verbose:
+      print " You will update this :"
+      print open(self.config.upgrade_file).read()
+
     if dry_run:
       return
 
-    self._upload(self.config.upgrade_file)
+    if get_yes_no("Do you want to continue? "):
+      self._upload(self.config.upgrade_file)
  
   def update(self, reboot=None, upgrade=None):
-    self.load()
     if reboot is None and upgrade is None:
       return 
     if not self.current_state.has_section('system'):
@@ -182,6 +195,20 @@ class Signature:
     self.current_state.write(current_state_file)
     current_state_file.close()
 
+  def get_signature_dict(self):
+    """ Convert Next state info into a dict """
+    map_dict = {}
+    for key in self.next_state.sections():
+      if key == "system":
+        continue
+      def clean_list(l):
+        return [x.strip() for x in l.split('\n') if x.strip() != '']
+      map_dict[key] = {}
+      for entry in self.next_state.options(key):
+        map_dict[key][entry] = clean_list(self.next_state.get(key, entry))
+
+    return map_dict
+
   def _read_state(self, state, name):
     """ Extract information from config file """
     if not state.has_section('system'):
@@ -196,10 +223,10 @@ class Signature:
     self.current_state = ConfigParser.RawConfigParser()
     self.current_state.read(self.config.srv_file)
 
-    self.next_state = ConfigParser.RawConfigParser()
+    self.next_state = ConfigParser.ConfigParser()
     self.next_state.read(self.download())
 
-    self.reboot = self._read_state(self.next_state, "upgrade")
+    self.reboot = self._read_state(self.next_state, "reboot")
     self.upgrade = self._read_state(self.next_state, "upgrade")
     self.last_reboot = self._read_state(self.current_state, "reboot")
     self.last_upgrade = self._read_state(self.current_state, "upgrade")

slapos/package/update.py

@@ -37,7 +37,7 @@ import subprocess as sub
 import sys
 import tempfile
 from signature import Signature
-
+from base_promise import BasePromise
 
 # create console handler and set level to warning
 ch = logging.StreamHandler()
@@ -97,48 +97,78 @@ class Upgrader:
     # add ch to logger
     self.logger.addHandler(ch)
 
-  def checkConsistency(self, *args, **kw):
-    print "CHECK CONSISTENCY %s" % ((args, kw),)
-
-  def run(self):
-    """
-    Will fetch information from web and update and/or reboot
-    machine if needed
-    """
+  def fixConsistency(self, signature, upgrade=0, reboot=0, boot=0, **kw):
+    print upgrade, reboot, boot
     today = datetime.date.today().isoformat()
+    if upgrade and boot:
+      signature.update(reboot=today, upgrade=today)
+    if upgrade:
+      signature.update(upgrade=today)
+    elif reboot:
+      signature.update(reboot=today)
+    else:
+      raise ValueError(
+        "You need upgrade and/or reboot when invoke fixConsistency!")
+
+    if upgrade:
+      pkgmanager = BasePromise()
+      configuration_dict = signature.get_signature_dict()
+      for entry in configuration_dict:
+         signature_list = configuration_dict[entry].get("signature-list")
+         if pkgmanager.matchSignatureList(signature_list):
+           print "Upgrade FOUND!!!! %s " % entry
+           upgrade_goal = configuration_dict[entry]
+           break
+
+      repository_tuple_list = []
+      for repository in upgrade_goal['repository-list']:
+        alias, url = repository.split("=")
+        repository_tuple_list.append((alias.strip(), url.strip()))
+
+      pkgmanager.update(repository_tuple_list, upgrade_goal['filter-package-list'])
+
+  def checkConsistency(self, fixit=0, **kw):
   
     # Get configuration
     signature = Signature(self.config)
-  
     signature.load()
   
     self.logger.debug("Expected Reboot early them %s" % signature.reboot)
     self.logger.debug("Expected Upgrade early them %s" % signature.upgrade)
     self.logger.debug("Last reboot : %s" % signature.last_reboot)
     self.logger.debug("Last upgrade : %s" % signature.last_upgrade)
-  
+
+    if signature.upgrade > datetime.date.today():
+      self.logger.debug("Upgrade will happens on %s" % signature.upgrade)
+      #return
+ 
     # Check if run for first time
     if signature.last_reboot is None:
-      if not self.config.dry_run:
-        signature.update(reboot=today, upgrade=today)
-  
-      # Purge repositories list and add new ones
-      self.checkConsistency(fixit=not self.config.dry_run)
+      if fixit:
+        # Purge repositories list and add new ones
+        self.fixConsistency(signature, upgrade=1, boot=1)
     else:
       if signature.last_upgrade < signature.upgrade:
         # Purge repositories list and add new ones
-        if not self.config.dry_run:
-          signature.update(upgrade=today)
-        self.checkConsistency(fixit=not self.config.dry_run)
+        if fixit:
+          self.fixConsistency(signature, upgrade=1)
       else:
         logger.info("Your system is up to date")
   
       if signature.last_reboot < signature.reboot:
         if not self.config.dry_run:
-          signature.update(reboot=today)
+          self.fixConsistency(signature, reboot=1)
         else:
           self.logger.debug("Dry run: Rebooting required.")
   
+
+  def run(self):
+    """
+    Will fetch information from web and update and/or reboot
+    machine if needed
+    """
+    self.checkConsistency(fixit=not self.config.dry_run)
+  
 def main():
   """Update computer and slapos"""
   usage = "usage: %s [options] " % sys.argv[0]
@@ -147,6 +177,5 @@ def main():
   upgrader.run()
   sys.exit()
 
-
 if __name__ == '__main__':
   main()

slapos/package/upload_key.py

@@ -36,6 +36,10 @@ import sys
 from update import Config
 from signature import Signature
 
+def do_upgrade(config):
+  signature = Signature(config)
+  signature.upload(dry_run=config.dry_run)
+
 
 class Parser(OptionParser):
   """
@@ -84,28 +88,10 @@ def get_yes_no(prompt):
     if answer.upper() in ['N', 'NO']:
         return False
 
-def new_upgrade(config):
-  signature = Signature(config)
-  signature.upload(dry_run=1) 
-
-  print " You will update this :"
-  print open(config.upgrade_file).read()
-  if not get_yes_no("Do you want to continue? "):
-    sys.exit(0)
-
-  if not config.dry_run:
-    print "Uploading..."
-    signature.upload()
-
 def main():
   """Upload file to update computer and slapos"""
   usage = "usage: [options] "
   # Parse arguments
   config = Config(Parser(usage=usage).check_args())
-  config.srv_file = "/srv/slapupdate"
-  new_upgrade(config)
+  do_upgrade(config)
   sys.exit()
-
-
-if __name__ == '__main__':
-  main()