Merge branch 'master' into 1.0

slapos/recipe/librecipe/generic.py

@@ -155,16 +155,11 @@ class GenericBaseRecipe(object):
       lines.append(dedent("""
           # Check for other instances
           pidfile=%s
-          if [ -e $pidfile ]; then
-            pid=$(cat $pidfile)
-            if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
-              echo "Already running with pid $pid."
-              exit 1
-            else
-              rm $pidfile
-            fi
+          if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
+            echo "Already running with pid $pid."
+            exit 1
           fi
-          echo $$ > $pidfile""" % (shlex.quote(pidfile), command)))
+          echo $$ > $pidfile""" % shlex.quote(pidfile)))
 
     lines.append(dedent('''
     # If the wrapped command uses a shebang, execute the referenced

software/slaprunner/common.cfg

@@ -54,7 +54,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = 8fedcc83e2ecb488910ddd8a1c82fe24
+md5sum = 66dba3331f49492d317e9c14d68d1bcb
 mode = 0644
 
 [template-runner-import-script]
@@ -69,7 +69,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = 8ae80f9a9d5523219e1c9065f1cab6d8
+md5sum = 673c30e5e7f9b7bb543f79465a56e43d
 mode = 0644
 
 [template-runner-export-script]
@@ -84,13 +84,13 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 8f4912ca04a650298c3c260689109c2e
+md5sum = 4b0ab39bc655ae0b865207147cb2e5bf
 mode = 0644
 
 [template-resilient]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = 160bf71d8a15f1359d80eb5586109781
+md5sum = 6373b2cf3c0cee78bd858959976271d3
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 

software/slaprunner/instance-resilient.cfg.jinja2

@@ -43,11 +43,6 @@ username = admin
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
 
-# XXX Monitoring Main Instane
-[monitor-instance-parameter]
-monitor-httpd-port = 8160
-cors-domains = {{ monitor_parameter.get('monitor-cors-domains', '') }}
-
 
 # Bubble up the parameters
 [request-runner]

software/slaprunner/instance-runner-export.cfg.in

@@ -35,6 +35,7 @@ parts +=
   httpd-graceful-wrapper
 ## Monitoring part
 ## Monitor for runner
+  monitor-base
   monitor-check-resilient-feed-file
   monitor-check-webrunner-internal-instance
 
@@ -66,7 +67,7 @@ instance-configuration =
 [monitor-check-resilient-feed-file]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
-rendered = $${monitor-directory:promises}/check-create-resilient-feed-files
+rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
 mode = 700
 context =
   key input_feed_directory directory:notifier-feeds

software/slaprunner/instance-runner-import.cfg.in

@@ -29,6 +29,8 @@ parts +=
 # have to repeat the next one, as it's not inherited from pbsready-import
   import-on-notification
 
+  monitor-base
+
 # For the needs of importer, we run the full slaprunner
 # In case both exporter and importer (aka main instance and clone instance)
 # run with the same IP (usually for testing purposes),

software/slaprunner/instance-runner.cfg

@@ -34,7 +34,7 @@ parts =
   custom-frontend-promise
 {% endif %}
 ## Monitoring part
-
+  monitor-base
   monitor-check-webrunner-internal-instance
 
 extends = ${monitor2-template:rendered}
@@ -308,9 +308,7 @@ path_access_log = $${directory:log}/httpd-access.log
 cert_file = $${ca-nginx:cert-file}
 key_file = $${ca-nginx:key-file}
 global_ip = $${slap-network-information:global-ipv6}
-global_port = 8386
-#httpd_port = $${monitor-parameters:port}
-#monitor_index = $${deploy-index:rendered}
+global_port = $${slap-parameter:slaprunner-httpd-port}
 working_directory = $${slaprunner:working-directory}
 dav_lock = $${directory:var}/WebDavLock
 htpasswd_file = $${monitor-httpd-conf-parameter:htpasswd-file}
@@ -441,7 +439,9 @@ return = site_url domain
 [request-httpd-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = SlapRunner httpd Frontend
+# XXX - Unfortunately, we still call webrunner httpd frontend "Monitor Frontend" otherwise 
+# buildout will ignore previous frontend that was created and create a new one (in case of upgrade)
+name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
@@ -469,7 +469,7 @@ backend_url = $${slaprunner:access-url}
 access_url = $${:url}/login
 url =  https://$${request-frontend:connection-domain}
 ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
-webdav_url = $${request-httpd-frontend:connection-secure_access}/shared/
+webdav_url = $${request-httpd-frontend:connection-secure_access}/share/
 public_url =  $${request-httpd-frontend:connection-secure_access}/public/
 git_public_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
 git_private_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
@@ -524,10 +524,14 @@ slapos-reference = master
 auto-deploy = false
 auto-deploy-instance = true
 autorun = false
-monitor-port = 9687
+slaprunner-httpd-port = $${:monitor-port}
+# XXX - for backward compatibility, monitor-port was for slaprunner httpd server
+monitor-port = 9684
 instance-name =
 monitor-cors-domains = 
 monitor-interface-url = 
+# XXX - define a new port for monitor here and use monitor-port for backward compatibility
+monitor-httpd-port = 8386
 
 [monitor-parameters]
 port = $${slap-parameter:monitor-port}
@@ -718,7 +722,7 @@ port = $${supervisord:port}
 
 # XXX Monitor 
 [monitor-instance-parameter]
-monitor-httpd-port = $${slap-parameter:monitor-port}
+monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
 {% if slapparameter_dict.get('name', '') -%}
 monitor-title = {{ slapparameter_dict['name'] }}
 {% endif -%}
@@ -732,8 +736,8 @@ password = {{ slapparameter_dict['monitor-password'] }}
 {% if slapparameter_dict.get('monitor-url-list', '') -%}
 opml-url-list = {{ slapparameter_dict['monitor-url-list'] }}
 {% endif -%}
+# check if not resilient runner
 {% if not slapparameter_dict.get('authorized-key', '') -%}
-# Parameter for resilient
 # Pass some parameter to dispay in monitoring interface
 instance-configuration =
   file recovery-code $${recovery-code:storage-path}
@@ -745,7 +749,7 @@ configuration-file-path = $${buildout:directory}/knowledge0.cfg
 [monitor-check-webrunner-internal-instance]
 recipe = slapos.recipe.template:jinja2
 template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
-rendered = $${monitor-directory:promises}/$${:filename}
+rendered = $${monitor-directory:reports}/$${:filename}
 filename = monitor-check-webrunner-internal-instance
 mode = 0744
 

software/test-slave-instance-deployment/software.cfg

@@ -142,5 +142,5 @@ setuptools = 0.9.8
 prettytable = 0.7.3-nxd001
 
 [networkcache]
-download-cache-url = http://download.shacache.org/
-download-dir-url = http://dir.shacache.org/
+download-cache-url = http://www.shacache.org/shacache
+download-dir-url = http://www.shacache.org/shadir
\ No newline at end of file

stack/monitor/buildout.cfg

@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = cd7f386fe1b5066d8508758249b408d3  
+md5sum = b6da7709da533c5c8280e5e103809c58  
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -130,13 +130,13 @@ depends =
 [monitor2-bin]
 <= monitor-template-script
 filename = monitor.py
-md5sum = 31beec15d3c3cd7979d04ecb834c439a
+md5sum = 6db3d9fcfff32c556f2bedee804ffb47
 
 [run-promise-py]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/scripts/run-promise.py
 rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 8ba8b661c55f2c5a379e9e42573be486
+md5sum = 5694920c5789f4d953cb5d41adfa7f8b
 mode = 0755
 context =
   raw python ${buildout:directory}/bin/${extra-eggs:interpreter}

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -70,7 +70,7 @@ etc = ${directory:etc}
 run = ${directory:monitor}/run
 #run = ${directory:scripts}
 promises = ${directory:monitor-promise}
-report = ${directory:monitor-report}
+reports = ${directory:monitor-report}
 pids = ${directory:run}/monitor
 cgi-bin = ${directory:monitor}/cgi-bin
 webdav = ${directory:monitor}/webdav
@@ -134,6 +134,7 @@ root-title = ${monitor-instance-parameter:root-instance-title}
 public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
 webdav-folder = ${monitor-directory:webdav}
+report-folder = ${monitor-directory:reports}
 web-folder = ${monitor-directory:web-dir}
 base-url = ${monitor-instance-parameter:monitor-base-url}
 monitor-hal-json = ${monitor-directory:public}/monitor.hal.json

stack/monitor/scripts/monitor.py

@@ -94,6 +94,7 @@ class Monitoring(object):
     self.collector_db  = config.get("monitor", "collector-db")
     self.collect_script = config.get("monitor", "collect-script")
     self.webdav_folder = config.get("monitor", "webdav-folder")
+    self.report_script_folder = config.get("monitor", "report-folder")
     self.webdav_url = '%s/share' % config.get("monitor", "base-url")
     self.public_url = '%s/public' % config.get("monitor", "base-url")
     self.status_history_folder = os.path.join(self.public_folder, 'history')
@@ -106,6 +107,7 @@ class Monitoring(object):
     self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
 
     self.config_folder = os.path.join(self.private_folder, 'config')
+    self.report_folder = os.path.join(self.private_folder, 'report')
 
     self.promise_dict = {}
     for promise_folder in self.promise_folder_list:
@@ -267,9 +269,11 @@ class Monitoring(object):
         raise
 
     self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
+    self.report_folder = os.path.join(self.report_folder, '.jio_documents')
     config_folder = os.path.join(self.config_folder, '.jio_documents')
     mkdirAll(self.data_folder)
     mkdirAll(config_folder)
+    mkdirAll(self.report_folder)
     try:
       os.symlink(os.path.join(self.private_folder, 'data'),
                   os.path.join(jio_private, 'data'))
@@ -281,6 +285,11 @@ class Monitoring(object):
     except OSError, e:
       if e.errno != os.errno.EEXIST:
         raise
+    try:
+      os.symlink(self.report_folder, os.path.join(jio_private, 'report'))
+    except OSError, e:
+      if e.errno != os.errno.EEXIST:
+        raise
 
   def makeConfigurationFiles(self):
     config_folder = os.path.join(self.config_folder, '.jio_documents')
@@ -385,6 +394,53 @@ class Monitoring(object):
     with open(self.monitor_hal_json, "w") as fp:
       json.dump(self.monitor_dict, fp)
 
+  def generateReportCronEntries(self):
+    cron_line_list = []
+    # We should add the possibility to modify this parameter later from monitor interface
+    report_frequency = "*/30 * * * *"
+
+    report_name_list = [name.replace('.report.json', '')
+      for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
+
+    for filename in os.listdir(self.report_script_folder):
+      report_script = os.path.join(self.report_script_folder, filename)
+      if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
+        report_name = os.path.splitext(filename)[0]
+        report_json_path = "%s.report.json" % report_name
+
+        report_cmd_line = [
+          report_frequency,
+          self.promise_runner,
+          '--pid_path "%s"' % os.path.join(self.service_pid_folder,
+            "%s.pid" % filename),
+          '--output "%s"' % os.path.join(self.report_folder,report_json_path),
+          '--promise_script "%s"' % report_script,
+          '--promise_name "%s"' % report_name,
+          '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
+          '--history_folder "%s"' % self.report_folder,
+          '--instance_name "%s"' % self.title,
+          '--hosting_name "%s"' % self.root_title,
+          '--promise_type "report"']
+
+        cron_line_list.append(' '.join(report_cmd_line))
+
+      if report_name in report_name_list:
+        report_name_list.pop(report_name_list.index(report_name))
+
+    # cleanup removed report json result
+    if report_name_list != []:
+      for report_name in report_name_list:
+        result_path = os.path.join(self.public_folder, '%s.report.json' % report_name)
+        if os.path.exists(result_path):
+          try:
+            os.unlink(result_path)
+          except OSError, e:
+            print "Error: Failed to delete %s" % result_path, str(e)
+            pass
+
+    with open(self.crond_folder + "/monitor-reports", "w") as freport:
+      freport.write("\n".join(cron_line_list))
+
   def generateServiceCronEntries(self):
     # XXX only if at least one configuration file is modified, then write in the cron
     #cron_line_list = ['PATH=%s\n' % os.environ['PATH']]
@@ -401,13 +457,13 @@ class Monitoring(object):
       promise_cmd_line = [
         softConfigGet(service_config, "service", "frequency") or "* * * * *",
         self.promise_runner,
-        '--pid_path %s' % os.path.join(self.service_pid_folder,
+        '--pid_path "%s"' % os.path.join(self.service_pid_folder,
           "%s.pid" % service_name),
-        '--output %s' % service_status_path,
-        '--promise_script %s' % promise["path"],
+        '--output "%s"' % service_status_path,
+        '--promise_script "%s"' % promise["path"],
         '--promise_name "%s"' % service_name,
         '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-        '--history_folder %s' % self.status_history_folder,
+        '--history_folder "%s"' % self.status_history_folder,
         '--instance_name "%s"' % self.title,
         '--hosting_name "%s"' % self.root_title]
 
@@ -442,6 +498,13 @@ class Monitoring(object):
       cronf.write(entry_line)
 
   def bootstrapMonitor(self):
+
+    # create symlinks from monitor.conf
+    self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
+    self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
+
+    self.configureFolders()
+
     # create symlinks from service configurations
     self.promise_items = self.promise_dict.items()
     for service_name, promise in self.promise_items:
@@ -457,12 +520,6 @@ class Monitoring(object):
                                       private_path_list.split(),
                                       service_name)
 
-    # create symlinks from monitor.conf
-    self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
-    self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
-
-    self.configureFolders()
-
     # generate monitor.json
     self.monitor_dict = {}
     self.generateMonitorHalJson()
@@ -474,6 +531,9 @@ class Monitoring(object):
     # put promises to a cron file
     self.generateServiceCronEntries()
 
+    # put report script to cron
+    self.generateReportCronEntries()
+
     # Generate parameters files and scripts
     self.makeConfigurationFiles()
 

stack/monitor/scripts/run-promise.py

@@ -24,15 +24,18 @@ def parseArguments():
                       help='Promise script to execute.')
   parser.add_argument('--promise_name',
                       help='Title to give to this promise.')
+  parser.add_argument('--promise_type',
+                      default='status',
+                      help='Type of promise to execute. [status, report].')
   parser.add_argument('--monitor_url',
                       help='Monitor Instance website URL.')
   parser.add_argument('--history_folder',
                       help='Path where old result file will be placed before generate a new json result file.')
   parser.add_argument('--instance_name',
-                      default='UNKNOW Software Instance',
+                      default='UNKNOWN Software Instance',
                       help='Software Instance name.')
   parser.add_argument('--hosting_name',
-                      default='UNKNOW Hosting Subscription',
+                      default='UNKNOWN Hosting Subscription',
                       help='Hosting Subscription name.')
 
   return parser.parse_args()
@@ -74,14 +77,16 @@ def main():
   updateStatusHistoryFolder(
     parser.promise_name,
     parser.output,
-    parser.history_folder
+    parser.history_folder,
+    parser.promise_type
   )
   with open(parser.output, "w") as outputfile:
     json.dump(status_json, outputfile)
   os.remove(parser.pid_path)
 
-def updateStatusHistoryFolder(name, status_file, history_folder):
+def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
   old_history_list = []
+  keep_item_amount = 25
   history_path = os.path.join(history_folder, name, '.jio_documents')
   if not os.path.exists(status_file):
     return
@@ -96,18 +101,19 @@ def updateStatusHistoryFolder(name, status_file, history_folder):
       else: raise
   with open(status_file, 'r') as sf:
     status_dict = json.loads(sf.read())
-    filename = '%s.status.json' % (
-      status_dict['start-date'].replace(' ', '_').replace(':', ''))
+    filename = '%s.%s.json' % (
+      status_dict['start-date'].replace(' ', '_').replace(':', ''),
+      promise_type)
 
   copyfile(status_file, os.path.join(history_path, filename))
-  # Don't let history foler grow too much, keep 40 files
+  # Don't let history foler grow too much, keep xx files
   file_list = filter(os.path.isfile,
-      glob.glob("%s/*.status.json" % history_path)
+      glob.glob("%s/*.%s.json" % (history_path, promise_type))
     )
   file_count = len(file_list)
-  if file_count > 40:
+  if file_count > keep_item_amount:
     file_list.sort(key=lambda x: os.path.getmtime(x))
-    while file_count > 40:
+    while file_count > keep_item_amount:
       to_delete = file_list.pop(0)
       try:
         os.unlink(to_delete)

stack/resilient/buildout.cfg

@@ -63,7 +63,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 028dfc01dfb0d738e1b4793f67b24e8c
+md5sum = cb7acac7ab41bf44c20d6d03bfad8217
 mode = 0644
 
 [template-replicated]

stack/resilient/instance-pull-backup.cfg.in

@@ -11,6 +11,7 @@ parts =
 
 
 ## Monitor for pbs
+  monitor-base
   monitor-check-resilient-feed-file
 
 extends = ${monitor2-template:rendered}
@@ -266,7 +267,7 @@ private-path-list +=
 [monitor-check-resilient-feed-file]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
-rendered = $${monitor-directory:promises}/check-create-resilient-feed-files
+rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
 mode = 700
 context =
   key input_feed_directory directory:notifier-feeds

stack/shacache-client.cfg

@@ -3,5 +3,5 @@
 networkcache-section = networkcache
 
 [networkcache]
-download-cache-url = http://download.shacache.org/
-download-dir-url = http://dir.shacache.org/
+download-cache-url = http://www.shacache.org/shacache
+download-dir-url = http://www.shacache.org/shadir

stack/slapos.cfg

@@ -194,8 +194,8 @@ uritemplate = 0.6
 zope.interface = 4.1.3
 
 [networkcache]
-download-cache-url = http://download.shacache.org/
-download-dir-url = http://dir.shacache.org/
+download-cache-url = http://www.shacache.org/shacache
+download-dir-url = http://www.shacache.org/shadir
 
 # signature certificates of the following uploaders.
 #   Romain Courteaud