utils.py 27.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
Marco Mariani's avatar
Marco Mariani committed
2
# vim: set et sts=2:
Marco Mariani's avatar
Marco Mariani committed
3
# pylint: disable-msg=W0311,C0301,C0103,C0111,W0141,W0142
4

5
import ConfigParser
6
import datetime
7
import json
Marco Mariani's avatar
Marco Mariani committed
8
import logging
Marco Mariani's avatar
Marco Mariani committed
9
import md5
10
import os
11
import sup_process
12 13
import re
import shutil
14
import thread
Marco Mariani's avatar
Marco Mariani committed
15 16
import time
import urllib
17
import xmlrpclib
Alain Takoudjou's avatar
Alain Takoudjou committed
18
from xml.dom import minidom
Marco Mariani's avatar
Marco Mariani committed
19 20 21 22

import xml_marshaller
from flask import jsonify

23
from slapos.runner.gittools import cloneRepo
24

Nicolas Wavrant's avatar
Nicolas Wavrant committed
25
from slapos.runner.process import Popen
Marco Mariani's avatar
Marco Mariani committed
26
from slapos.htpasswd import HtpasswdFile
Marco Mariani's avatar
Marco Mariani committed
27
import slapos.slap
28
from slapos.grid.utils import md5digest
Łukasz Nowak's avatar
Łukasz Nowak committed
29

30
# Setup default flask (werkzeug) parser
Marco Mariani's avatar
Marco Mariani committed
31

32
logger = logging.getLogger('werkzeug')
Łukasz Nowak's avatar
Łukasz Nowak committed
33

34
TRUE_VALUES = (1, '1', True, 'true', 'True')
35

36 37 38 39 40 41 42
html_escape_table = {
  "&": "&",
  '"': """,
  "'": "'",
  ">": ">",
  "<": "&lt;",
}
43

44 45 46 47
def getBuildAndRunParams(config):
  json_file = os.path.join(config['etc_dir'], 'config.json')
  json_params = json.load(open(json_file))
  return json_params
Marco Mariani's avatar
Marco Mariani committed
48

49

50
def saveBuildAndRunParams(config, params):
51 52 53
  """XXX-Nico parameters have to be correct.
  Works like that because this function do not care
  about how you got the parameters"""
54 55 56
  json_file = os.path.join(config['etc_dir'], 'config.json')
  open(json_file, "w").write(json.dumps(params))

57

58 59
def html_escape(text):
  """Produce entities within text."""
Marco Mariani's avatar
Marco Mariani committed
60
  return "".join(html_escape_table.get(c, c) for c in text)
61

Marco Mariani's avatar
Marco Mariani committed
62

63 64 65 66
def getSession(config):
  """
  Get the session data of current user.
  Returns:
67
    a list of user information or None if the file does not exist.
68
  """
69
  user_path = os.path.join(config['etc_dir'], '.users')
70
  if os.path.exists(user_path):
71
    return open(user_path).read().split(';')
72

Marco Mariani's avatar
Marco Mariani committed
73

74
def saveSession(config, account):
75 76 77 78 79 80 81 82 83 84 85
  """
  Save account information for the current user

  Args:
    config: Slaprunner configuration
    session: Flask session
    account: New session data to be save

  Returns:
    True if all goes well or str (error message) if fail
  """
86
  # XXX Cedric LN hardcoded path for files
87
  user = os.path.join(config['etc_dir'], '.users')
88
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
89 90
  backup = False
  try:
91
    if os.path.exists(user):
92
      #backup previous data
93
      data = open(user).read()
Marco Mariani's avatar
Marco Mariani committed
94
      open('%s.back' % user, 'w').write(data)
95
      backup = True
96 97
      if not account[1]:
        account[1] = data.split(';')[1]
98 99
    #save new account data
    open(user, 'w').write((';'.join(account)).encode("utf-8"))
100 101
    # Htpasswd file for cloud9
    # XXX Cedric Le N order of account list values suppose to be fixed
102
    # Remove former file to avoid outdated accounts
103 104
    if os.path.exists(htpasswdfile):
      os.remove(htpasswdfile)
105
    passwd = HtpasswdFile(htpasswdfile, create=True)
106 107
    passwd.update(account[0], account[1])
    passwd.save()
108
    return True
Marco Mariani's avatar
Marco Mariani committed
109
  except Exception as e:
110 111 112
    try:
      if backup:
        os.remove(user)
Marco Mariani's avatar
Marco Mariani committed
113
        os.rename('%s.back' % user, user)
114 115 116
    except:
      pass
    return str(e)
Łukasz Nowak's avatar
Łukasz Nowak committed
117

Marco Mariani's avatar
Marco Mariani committed
118

119 120 121 122 123 124 125 126 127
def getRcode(config):
  parser = ConfigParser.ConfigParser()
  try:
    parser.read(config['knowledge0_cfg'])
    return parser.get('public', 'recovery-code')
  except (ConfigParser.NoSectionError, IOError) as e:
    return None


128 129 130 131 132 133 134 135 136 137
def createNewUser(config, name, passwd):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
  if os.path.exists(htpasswdfile):
    htpasswd = HtpasswdFile(htpasswdfile)
    htpasswd.update(name, passwd)
    htpasswd.save()
    return True
  return False


138 139 140 141 142 143
def getCurrentSoftwareReleaseProfile(config):
  """
  Returns used Software Release profile as a string.
  """
  try:
    software_folder = open(
144
        os.path.join(config['etc_dir'], ".project")).read().rstrip()
145 146
    return realpath(
        config, os.path.join(software_folder, config['software_profile']))
147
  # XXXX No Comments
148
  except:
149
    return ''
150

Marco Mariani's avatar
Marco Mariani committed
151

152 153 154 155
def requestInstance(config, software_type=None):
  """
  Request the main instance of our environment
  """
156
  software_type_path = os.path.join(config['etc_dir'], ".software_type.xml")
157
  if software_type:
158 159 160
    # Write it to conf file for later use
    open(software_type_path, 'w').write(software_type)
  elif os.path.exists(software_type_path):
161
    software_type = open(software_type_path).read().rstrip()
162 163
  else:
    software_type = 'default'
164 165 166 167 168 169 170 171

  slap = slapos.slap.slap()
  profile = getCurrentSoftwareReleaseProfile(config)
  slap.initializeConnection(config['master_url'])

  param_path = os.path.join(config['etc_dir'], ".parameter.xml")
  xml_result = readParameters(param_path)
  partition_parameter_kw = None
Marco Mariani's avatar
Marco Mariani committed
172
  if type(xml_result) != type('') and 'instance' in xml_result:
173 174 175 176 177 178
    partition_parameter_kw = xml_result['instance']

  return slap.registerOpenOrder().request(
      profile,
      partition_reference=getSoftwareReleaseName(config),
      partition_parameter_kw=partition_parameter_kw,
179
      software_type=software_type,
180 181 182 183
      filter_kw=None,
      state=None,
      shared=False)

Marco Mariani's avatar
Marco Mariani committed
184

185
def updateProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
186 187 188 189
  """
  Configure Slapos Node computer and partitions.
  Send current Software Release to Slapproxy for compilation and deployment.
  """
190
  startProxy(config)
Łukasz Nowak's avatar
Łukasz Nowak committed
191 192 193
  if not os.path.exists(config['instance_root']):
    os.mkdir(config['instance_root'])
  slap = slapos.slap.slap()
194
  profile = getCurrentSoftwareReleaseProfile(config)
195

Łukasz Nowak's avatar
Łukasz Nowak committed
196
  slap.initializeConnection(config['master_url'])
197
  slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
Łukasz Nowak's avatar
Łukasz Nowak committed
198 199 200
  computer = slap.registerComputer(config['computer_id'])
  prefix = 'slappart'
  slap_config = {
Marco Mariani's avatar
Marco Mariani committed
201 202 203 204 205 206 207 208
    'address': config['ipv4_address'],
    'instance_root': config['instance_root'],
    'netmask': '255.255.255.255',
    'partition_list': [],
    'reference': config['computer_id'],
    'software_root': config['software_root']
  }

Łukasz Nowak's avatar
Łukasz Nowak committed
209 210 211 212 213 214
  for i in xrange(0, int(config['partition_amount'])):
    partition_reference = '%s%s' % (prefix, i)
    partition_path = os.path.join(config['instance_root'], partition_reference)
    if not os.path.exists(partition_path):
      os.mkdir(partition_path)
    os.chmod(partition_path, 0750)
Marco Mariani's avatar
Marco Mariani committed
215 216 217 218 219 220 221 222 223 224 225 226 227
    slap_config['partition_list'].append({
                                           'address_list': [
                                              {
                                                'addr': config['ipv4_address'],
                                                'netmask': '255.255.255.255'
                                              }, {
                                                'addr': config['ipv6_address'],
                                                'netmask': 'ffff:ffff:ffff::'
                                              },
                                           ],
                                           'path': partition_path,
                                           'reference': partition_reference,
                                           'tap': {'name': partition_reference}})
Marco Mariani's avatar
Marco Mariani committed
228
  computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
229
  return True
Łukasz Nowak's avatar
Łukasz Nowak committed
230

231

232
def updateInstanceParameter(config, software_type=None):
Alain Takoudjou's avatar
Alain Takoudjou committed
233 234 235 236 237 238 239
  """
  Reconfigure Slapproxy to re-deploy current Software Instance with parameters.

  Args:
    config: Slaprunner configuration.
    software_type: reconfigure Software Instance with software type.
  """
240
  time.sleep(1)
241
  if not (updateProxy(config) and requestInstance(config, software_type)):
242 243
    return False

244

Łukasz Nowak's avatar
Łukasz Nowak committed
245
def startProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
246
  """Start Slapproxy server"""
247
  if sup_process.isRunning(config, 'slapproxy'):
248
    return
249 250 251 252
  try:
    sup_process.runProcess(config, "slapproxy")
  except xmlrpclib.Fault:
    pass
253
  time.sleep(4)
Łukasz Nowak's avatar
Łukasz Nowak committed
254 255 256


def stopProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
257
  """Stop Slapproxy server"""
258
  pass
Łukasz Nowak's avatar
Łukasz Nowak committed
259 260 261


def removeProxyDb(config):
262
  """Remove Slapproxy database, this is used to initialize proxy for example when
Alain Takoudjou's avatar
Alain Takoudjou committed
263
    configuring new Software Release"""
Łukasz Nowak's avatar
Łukasz Nowak committed
264 265 266
  if os.path.exists(config['database_uri']):
    os.unlink(config['database_uri'])

Marco Mariani's avatar
Marco Mariani committed
267

268
def isSoftwareRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
269
  """
270
    Return True if slapos is still running and false if slapos if not
Alain Takoudjou's avatar
Alain Takoudjou committed
271
  """
272
  return sup_process.isRunning(config, 'slapgrid-sr')
Łukasz Nowak's avatar
Łukasz Nowak committed
273 274


275 276 277 278 279 280 281
def slapgridResultToFile(config, step, returncode, datetime):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  result = {'last_build':datetime, 'success':returncode}
  open(file, "w").write(json.dumps(result))


282 283 284 285 286 287 288 289 290 291
def getSlapgridResult(config, step):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  if os.path.exists(file):
    result = json.loads(open(file, "r").read())
  else:
    result = {'last_build': 0, 'success':-1}
  return result


292 293 294 295 296 297
def waitProcess(config, process, step):
  process.wait()
  date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  slapgridResultToFile(config, step, process.returncode, date)


298
def runSoftwareWithLock(config, lock=False):
Alain Takoudjou's avatar
Alain Takoudjou committed
299 300 301 302
  """
    Use Slapgrid to compile current Software Release and wait until
    compilation is done
  """
303 304
  if sup_process.isRunning(config, 'slapgrid-sr'):
    return 1
305 306 307 308 309

  if not os.path.exists(config['software_root']):
    os.mkdir(config['software_root'])
  stopProxy(config)
  startProxy(config)
310 311 312
  # XXX Hackish and unreliable
  if os.path.exists(config['software_log']):
    os.remove(config['software_log'])
313
  if not updateProxy(config):
314
    return 1
315 316 317 318
  try:
    sup_process.runProcess(config, "slapgrid-sr")
    if lock:
      sup_process.waitForProcessEnd(config, "slapgrid-sr")
319 320
    #Saves the current compile software for re-use
    config_SR_folder(config)
321
    return sup_process.returnCode(config, "slapgrid-sr")
322
  except xmlrpclib.Fault:
323
    return 1
324

Łukasz Nowak's avatar
Łukasz Nowak committed
325

326
def config_SR_folder(config):
Marco Mariani's avatar
Marco Mariani committed
327 328
  """Create a symbolik link for each folder in software folder. That allows
    the user to customize software release folder"""
329
  config_name = 'slaprunner.config'
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
  def link_to_folder(name, folder):
    destination = os.path.join(config['software_link'], name)
    source = os.path.join(config['software_root'], folder)
    cfg = os.path.join(destination, config_name)
        #create symlink
    if os.path.lexists(destination):
      os.remove(destination)
    os.symlink(source, destination)
        #write config file
    if os.path.exists(source):
      with open(cfg, 'w') as cf:
        cf.write(current_project + '#' + folder)

  # First create the link for current project
  current_project = open(os.path.join(config['etc_dir'], ".project")).read()
  profile = getCurrentSoftwareReleaseProfile(config)
  if current_project[-1] == '/':
     current_project = current_project[:-1]
  name = current_project.split('/')[-1]
  md5sum = md5digest(profile)
  link_to_folder(name, md5sum)
  # check other links
  # XXX-Marco do not shadow 'list'
  list = []
354 355 356
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
357
      cfg = open(cfg_path).read().split("#")
358
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
359
        continue  # there is a broken config file
360
      list.append(cfg[1])
361 362 363 364
  if os.path.exists(config['software_root']):
    folder_list = os.listdir(config['software_root'])
  else:
    return
365
  if not folder_list:
366 367 368
    return
  for folder in folder_list:
    if folder in list:
Marco Mariani's avatar
Marco Mariani committed
369
      continue  # this folder is already registered
370
    else:
371
      link_to_folder(folder, folder)
Marco Mariani's avatar
Marco Mariani committed
372

373 374 375 376 377 378 379
def loadSoftwareRList(config):
  """Return list (of dict) of Software Release from symbolik SR folder"""
  list = []
  config_name = 'slaprunner.config'
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
380
      cfg = open(cfg_path).read().split("#")
381
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
382
        continue  # there is a broken config file
383 384
      list.append(dict(md5=cfg[1], path=cfg[0], title=path))
  return list
Łukasz Nowak's avatar
Łukasz Nowak committed
385

Marco Mariani's avatar
Marco Mariani committed
386

387
def isInstanceRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
388
  """
389
    Return True if slapos is still running and False otherwise
Alain Takoudjou's avatar
Alain Takoudjou committed
390
  """
391
  return sup_process.isRunning(config, 'slapgrid-cp')
Łukasz Nowak's avatar
Łukasz Nowak committed
392 393


394
def runInstanceWithLock(config, lock=False):
Alain Takoudjou's avatar
Alain Takoudjou committed
395 396 397 398
  """
    Use Slapgrid to deploy current Software Release and wait until
    deployment is done.
  """
399 400
  if sup_process.isRunning(config, 'slapgrid-cp'):
    return 1
401 402

  startProxy(config)
403 404 405
  # XXX Hackish and unreliable
  if os.path.exists(config['instance_log']):
    os.remove(config['instance_log'])
406
  if not (updateProxy(config) and requestInstance(config)):
407
    return 1
408 409 410 411
  try:
    sup_process.runProcess(config, "slapgrid-cp")
    if lock:
      sup_process.waitForProcessEnd(config, "slapgrid-cp")
412
    return sup_process.returnCode(config, "slapgrid-cp")
413
  except xmlrpclib.Fault:
414
    return 1
415

Łukasz Nowak's avatar
Łukasz Nowak committed
416

Alain Takoudjou's avatar
Alain Takoudjou committed
417 418 419 420 421 422 423 424 425 426 427 428
def getProfilePath(projectDir, profile):
  """
  Return the path of the current Software Release `profile`

  Args:
    projectDir: Slaprunner workspace location.
    profile: file to search into the workspace.

  Returns:
    String, path of current Software Release profile
  """
  if not os.path.exists(os.path.join(projectDir, ".project")):
429
    return False
Alain Takoudjou's avatar
Alain Takoudjou committed
430
  projectFolder = open(os.path.join(projectDir, ".project")).read()
431
  return os.path.join(projectFolder, profile)
Łukasz Nowak's avatar
Łukasz Nowak committed
432

Marco Mariani's avatar
Marco Mariani committed
433

Łukasz Nowak's avatar
Łukasz Nowak committed
434
def getSlapStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
435
  """Return all Slapos Partitions with associate information"""
Łukasz Nowak's avatar
Łukasz Nowak committed
436 437 438 439 440 441 442
  slap = slapos.slap.slap()
  slap.initializeConnection(config['master_url'])
  partition_list = []
  computer = slap.registerComputer(config['computer_id'])
  try:
    for partition in computer.getComputerPartitionList():
      # Note: Internal use of API, as there is no reflexion interface in SLAP
443
      partition_list.append((partition.getId(), partition._connection_dict.copy()))
Łukasz Nowak's avatar
Łukasz Nowak committed
444 445
  except Exception:
    pass
446 447 448
  if partition_list:
    for i in xrange(0, int(config['partition_amount'])):
      slappart_id = '%s%s' % ("slappart", i)
Marco Mariani's avatar
Marco Mariani committed
449
      if not [x[0] for x in partition_list if slappart_id == x[0]]:
450
        partition_list.append((slappart_id, []))
Łukasz Nowak's avatar
Łukasz Nowak committed
451 452
  return partition_list

Marco Mariani's avatar
Marco Mariani committed
453

Łukasz Nowak's avatar
Łukasz Nowak committed
454
def svcStopAll(config):
455
  """Stop all Instance processes on this computer"""
456 457 458 459 460
  try:
    return Popen([config['supervisor'], config['configuration_file_path'],
                  'shutdown']).communicate()[0]
  except:
    pass
Łukasz Nowak's avatar
Łukasz Nowak committed
461

Marco Mariani's avatar
Marco Mariani committed
462

463
def removeInstanceRoot(config):
464
  """Clean instance directory and stop all its running processes"""
465 466
  if os.path.exists(config['instance_root']):
    svcStopAll(config)
Marco Mariani's avatar
Marco Mariani committed
467
    for root, dirs, _ in os.walk(config['instance_root']):
468
      for fname in dirs:
469
        fullPath = os.path.join(root, fname)
Marco Mariani's avatar
Marco Mariani committed
470
        if not os.access(fullPath, os.W_OK):
471 472
          # Some directories may be read-only, preventing to remove files in it
          os.chmod(fullPath, 0744)
473 474
    shutil.rmtree(config['instance_root'])

Marco Mariani's avatar
Marco Mariani committed
475

Łukasz Nowak's avatar
Łukasz Nowak committed
476
def getSvcStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
477
  """Return all Softwares Instances process Information"""
478
  result = Popen([config['supervisor'], config['configuration_file_path'],
479
                  'status']).communicate()[0]
480
  regex = "(^unix:.+\.socket)|(^error:)|(^watchdog).*$"
481 482
  supervisord = []
  for item in result.split('\n'):
483
    if item.strip() != "":
Marco Mariani's avatar
Marco Mariani committed
484
      if re.search(regex, item, re.IGNORECASE) is None:
485
        supervisord.append(re.split('[\s,]+', item))
486 487
  return supervisord

Marco Mariani's avatar
Marco Mariani committed
488

489
def getSvcTailProcess(config, process):
490
  """Get log for the specified process
Alain Takoudjou's avatar
Alain Takoudjou committed
491 492 493

  Args:
    config: Slaprunner configuration
494
    process: process name. this value is passed to supervisord.
Alain Takoudjou's avatar
Alain Takoudjou committed
495 496 497
  Returns:
    a string that contains the log of the process.
  """
498
  return Popen([config['supervisor'], config['configuration_file_path'],
499 500
                "tail", process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
501

502
def svcStartStopProcess(config, process, action):
Alain Takoudjou's avatar
Alain Takoudjou committed
503 504 505 506 507 508 509
  """Send start or stop process command to supervisord

  Args:
    config: Slaprunner configuration.
    process: process to start or stop.
    action: current state which is used to generate the new process state.
  """
Marco Mariani's avatar
Marco Mariani committed
510 511 512 513 514 515 516
  cmd = {
    'RESTART': 'restart',
    'STOPPED': 'start',
    'RUNNING': 'stop',
    'EXITED': 'start',
    'STOP': 'stop'
  }
517
  return Popen([config['supervisor'], config['configuration_file_path'],
518 519
                cmd[action], process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
520

521 522
def listFolder(config, path):
  """Return the list of folder into path
Alain Takoudjou's avatar
Alain Takoudjou committed
523 524

  Agrs:
525
    path: path of the directory to list
Alain Takoudjou's avatar
Alain Takoudjou committed
526 527 528
  Returns:
    a list that contains each folder name.
  """
529 530 531 532 533 534 535 536
  folderList = []
  folder = realpath(config, path)
  if folder:
    path_list = sorted(os.listdir(folder), key=str.lower)
    for elt in path_list:
      if os.path.isdir(os.path.join(folder, elt)):
        folderList.append(elt)
  return folderList
537

Marco Mariani's avatar
Marco Mariani committed
538

539
def configNewSR(config, projectpath):
Alain Takoudjou's avatar
Alain Takoudjou committed
540 541 542 543 544 545 546 547
  """Configure a Software Release as current Software Release

  Args:
    config: slaprunner configuration
    projectpath: path of the directory that contains the software realease to configure
  Returns:
    True if all is done well, otherwise return false.
  """
548 549
  folder = realpath(config, projectpath)
  if folder:
550 551
    sup_process.stopProcess(config, 'slapgrid-cp')
    sup_process.stopProcess(config, 'slapgrid-sr')
552 553 554 555
    stopProxy(config)
    removeProxyDb(config)
    startProxy(config)
    removeInstanceRoot(config)
556
    param_path = os.path.join(config['etc_dir'], ".parameter.xml")
557 558
    if os.path.exists(param_path):
      os.remove(param_path)
559
    open(os.path.join(config['etc_dir'], ".project"), 'w').write(projectpath)
560 561 562 563
    return True
  else:
    return False

Marco Mariani's avatar
Marco Mariani committed
564

565
def newSoftware(folder, config, session):
Alain Takoudjou's avatar
Alain Takoudjou committed
566 567 568 569 570 571 572
  """
  Create a new Software Release folder with default profiles

  Args:
    folder: directory of the new software release
    config: slraprunner configuration
    session: Flask session directory"""
573 574
  json = ""
  code = 0
575
  basedir = config['etc_dir']
576
  try:
577 578 579
    folderPath = realpath(config, folder, check_exist=False)
    if folderPath and not os.path.exists(folderPath):
      os.mkdir(folderPath)
580 581 582 583
      #load software.cfg and instance.cfg from http://git.erp5.org
      software = "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/lamp-template/software.cfg"
      softwareContent = ""
      try:
584
        softwareContent = urllib.urlopen(software).read()
585
      except:
586 587
        #Software.cfg and instance.cfg content will be empty
        pass
588
      open(os.path.join(folderPath, config['software_profile']), 'w').write(softwareContent)
589 590
      open(os.path.join(folderPath, config['instance_profile']), 'w').write("")
      open(os.path.join(basedir, ".project"), 'w').write(folder + "/")
591 592 593 594 595 596
      #Clean sapproxy Database
      stopProxy(config)
      removeProxyDb(config)
      startProxy(config)
      #Stop runngin process and remove existing instance
      removeInstanceRoot(config)
597 598 599
      session['title'] = getProjectTitle(config)
      code = 1
    else:
Marco Mariani's avatar
Marco Mariani committed
600
      json = "Bad folder or Directory '%s' already exist, please enter a new name for your software" % folder
Marco Mariani's avatar
Marco Mariani committed
601
  except Exception as e:
Marco Mariani's avatar
Marco Mariani committed
602
    json = "Can not create your software, please try again! : %s " % e
603 604
    if os.path.exists(folderPath):
      shutil.rmtree(folderPath)
605 606
  return jsonify(code=code, result=json)

Marco Mariani's avatar
Marco Mariani committed
607

608
def checkSoftwareFolder(path, config):
Alain Takoudjou's avatar
Alain Takoudjou committed
609
  """Check id `path` is a valid Software Release folder"""
610 611
  realdir = realpath(config, path)
  if realdir and os.path.exists(os.path.join(realdir, config['software_profile'])):
612 613 614
    return jsonify(result=path)
  return jsonify(result="")

Marco Mariani's avatar
Marco Mariani committed
615

616
def getProjectTitle(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
617
  """Generate the name of the current software Release (for slaprunner UI)"""
618
  conf = os.path.join(config['etc_dir'], ".project")
619
  if os.path.exists(conf):
620
    project = open(conf, "r").read().split("/")
Marco Mariani's avatar
Marco Mariani committed
621 622
    software = project[-2]
    return '%s (%s)' % (software, '/'.join(project[:-2]))
623 624
  return "No Profile"

Marco Mariani's avatar
Marco Mariani committed
625

626
def getSoftwareReleaseName(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
627
  """Get the name of the current Software Release"""
628
  sr_profile = os.path.join(config['etc_dir'], ".project")
629 630
  if os.path.exists(sr_profile):
    project = open(sr_profile, "r").read().split("/")
Marco Mariani's avatar
Marco Mariani committed
631
    software = project[-2]
632 633 634
    return software.replace(' ', '_')
  return "No_name"

Marco Mariani's avatar
Marco Mariani committed
635

636 637
def removeSoftwareByName(config, md5, folderName):
  """Remove all content of the software release specified by md5
Alain Takoudjou's avatar
Alain Takoudjou committed
638 639 640

  Args:
    config: slaprunner configuration
641 642
    foldername: the link name given to the software release
    md5: the md5 filename given by slapgrid to SR folder"""
643
  if isSoftwareRunning(config) or isInstanceRunning(config):
644
    raise Exception("Software installation or instantiation in progress, cannot remove")
645 646
  path = os.path.join(config['software_root'], md5)
  linkpath = os.path.join(config['software_link'], folderName)
647
  if not os.path.exists(path):
648
    raise Exception("Cannot remove software Release: No such file or directory")
649 650
  if not os.path.exists(linkpath):
    raise Exception("Cannot remove software Release: No such file or directory %s" %
Marco Mariani's avatar
Marco Mariani committed
651
                    ('software_root/' + folderName))
652
  svcStopAll(config)
653
  os.unlink(linkpath)
654
  shutil.rmtree(path)
655
  return loadSoftwareRList(config)
656

Marco Mariani's avatar
Marco Mariani committed
657

658 659 660 661 662 663 664 665 666 667 668 669
def tail(f, lines=20):
  """
  Returns the last `lines` lines of file `f`. It is an implementation of tail -f n.
  """
  BUFSIZ = 1024
  f.seek(0, 2)
  bytes = f.tell()
  size = lines + 1
  block = -1
  data = []
  while size > 0 and bytes > 0:
      if bytes - BUFSIZ > 0:
670 671 672 673
          # Seek back one whole BUFSIZ
          f.seek(block * BUFSIZ, 2)
          # read BUFFER
          data.insert(0, f.read(BUFSIZ))
674
      else:
675
          # file too small, start from begining
Marco Mariani's avatar
Marco Mariani committed
676
          f.seek(0, 0)
677 678
          # only read what was not read
          data.insert(0, f.read(bytes))
679 680 681 682
      linesFound = data[0].count('\n')
      size -= linesFound
      bytes -= BUFSIZ
      block -= 1
Marco Mariani's avatar
Marco Mariani committed
683
  return '\n'.join(''.join(data).splitlines()[-lines:])
684

Marco Mariani's avatar
Marco Mariani committed
685

686
def readFileFrom(f, lastPosition, limit=20000):
687 688 689
  """
  Returns the last lines of file `f`, from position lastPosition.
  and the last position
690
  limit = max number of characters to read
691 692 693
  """
  BUFSIZ = 1024
  f.seek(0, 2)
Marco Mariani's avatar
Marco Mariani committed
694
  # XXX-Marco do now shadow 'bytes'
695 696 697 698
  bytes = f.tell()
  block = -1
  data = ""
  length = bytes
Marco Mariani's avatar
Marco Mariani committed
699 700
  truncated = False  # True if a part of log data has been truncated
  if (lastPosition <= 0 and length > limit) or (length - lastPosition > limit):
701
    lastPosition = length - limit
702
    truncated = True
703 704
  size = bytes - lastPosition
  while bytes > lastPosition:
Marco Mariani's avatar
Marco Mariani committed
705
    if abs(block * BUFSIZ) <= size:
706 707 708 709
      # Seek back one whole BUFSIZ
      f.seek(block * BUFSIZ, 2)
      data = f.read(BUFSIZ) + data
    else:
Marco Mariani's avatar
Marco Mariani committed
710
      margin = abs(block * BUFSIZ) - size
711
      if length < BUFSIZ:
Marco Mariani's avatar
Marco Mariani committed
712
        f.seek(0, 0)
713
      else:
714 715
        seek = block * BUFSIZ + margin
        f.seek(seek, 2)
716 717 718 719
      data = f.read(BUFSIZ - margin) + data
    bytes -= BUFSIZ
    block -= 1
  f.close()
Marco Mariani's avatar
Marco Mariani committed
720 721 722 723 724 725
  return {
    'content': data,
    'position': length,
    'truncated': truncated
  }

726

727 728 729 730
def isText(file):
  """Return True if the mimetype of file is Text"""
  if not os.path.exists(file):
    return False
Marco Mariani's avatar
Marco Mariani committed
731
  text_range = ''.join(map(chr, [7, 8, 9, 10, 12, 13, 27] + range(0x20, 0x100)))
732 733 734
  is_binary_string = lambda bytes: bool(bytes.translate(None, text_range))
  try:
    return not is_binary_string(open(file).read(1024))
735 736
  except:
    return False
737

Marco Mariani's avatar
Marco Mariani committed
738

739 740
def md5sum(file):
  """Compute md5sum of `file` and return hexdigest value"""
741
  # XXX-Marco: returning object or False boolean is an anti-pattern. better to return object or None
742 743 744 745
  if os.path.isdir(file):
    return False
  try:
    fh = open(file, 'rb')
Marco Mariani's avatar
Marco Mariani committed
746
    m = md5.md5()
747 748 749
    while True:
      data = fh.read(8192)
      if not data:
750
        break
751 752
      m.update(data)
    return m.hexdigest()
753
  except:
754 755
    return False

Marco Mariani's avatar
Marco Mariani committed
756

757
def realpath(config, path, check_exist=True):
758 759 760 761
  """
  Get realpath of path or return False if user is not allowed to access to
  this file.
  """
762 763
  split_path = path.split('/')
  key = split_path[0]
Marco Mariani's avatar
Marco Mariani committed
764 765 766 767
  allow_list = {
    'software_root': config['software_root'],
    'instance_root': config['instance_root'],
    'workspace': config['workspace'],
768
    'runner_workdir': config['runner_workdir'],
Marco Mariani's avatar
Marco Mariani committed
769 770 771
    'software_link': config['software_link']
  }
  if key not in allow_list:
772
    return ''
Marco Mariani's avatar
Marco Mariani committed
773 774 775 776 777

  del split_path[0]
  path = os.path.join(allow_list[key], *split_path)
  if check_exist:
    if os.path.exists(path):
778
      return path
Marco Mariani's avatar
Marco Mariani committed
779
    else:
780
      return ''
Marco Mariani's avatar
Marco Mariani committed
781 782 783
  else:
    return path

784 785

def readParameters(path):
Alain Takoudjou's avatar
Alain Takoudjou committed
786 787 788 789 790 791
  """Read Instance parameters stored into a local file.

  Agrs:
    path: path of the xml file that contains parameters

  Return:
Marco Mariani's avatar
Marco Mariani committed
792
    a dictionary of instance parameters."""
793 794
  if os.path.exists(path):
    try:
Alain Takoudjou's avatar
Alain Takoudjou committed
795
      xmldoc = minidom.parse(path)
Alain Takoudjou's avatar
Alain Takoudjou committed
796
      obj = {}
797
      for elt in xmldoc.childNodes:
Marco Mariani's avatar
Marco Mariani committed
798
        sub_obj = {}
799 800
        for subnode in elt.childNodes:
          if subnode.nodeType != subnode.TEXT_NODE:
Marco Mariani's avatar
Marco Mariani committed
801
            sub_obj[str(subnode.getAttribute('id'))] = subnode.childNodes[0].data  # .decode('utf-8').decode('utf-8')
Marco Mariani's avatar
Marco Mariani committed
802 803
            obj[str(elt.tagName)] = sub_obj
      return obj
804 805 806
    except Exception, e:
      return str(e)
  else:
Marco Mariani's avatar
Marco Mariani committed
807
    return "No such file or directory: %s" % path
Nicolas Wavrant's avatar
Nicolas Wavrant committed
808

809

810 811 812 813
def isSoftwareReleaseReady(config):
  """Return 1 if the Software Release has
  correctly been deployed, 0 if not,
  and 2 if it is currently deploying"""
814 815
  auto_deploy = config['auto_deploy'] in TRUE_VALUES
  auto_run = config['autorun'] in TRUE_VALUES
816
  project = os.path.join(config['etc_dir'], '.project')
817
  if not ( os.path.exists(project) and (auto_run or auto_deploy) ):
818
    return "0"
819
  path = open(project, 'r').readline().strip()
820 821 822 823
  software_name = path
  if software_name[-1] == '/':
    software_name = software_name[:-1]
  software_name = software_name.split('/')[-1]
824
  updateInstanceParameter(config)
825
  config_SR_folder(config)
826
  if os.path.exists(os.path.join(config['runner_workdir'],
827
      'softwareLink', software_name, '.completed')):
828
    if auto_run:
829
      runSlapgridUntilSuccess(config, 'instance')
830 831 832 833
    return "1"
  else:
    if isSoftwareRunning(config):
      return "2"
834
    elif auto_deploy:
835
      runSoftwareWithLock(config)
836 837
      config_SR_folder(config)
      time.sleep(15)
838
      if auto_run:
839
        runSlapgridUntilSuccess(config, 'instance')
840 841 842
      return "2"
    else:
      return "0"
Nicolas Wavrant's avatar
Nicolas Wavrant committed
843

844

845
def cloneDefaultGit(config):
846
  """Test if the default git has been downloaded yet
Nicolas Wavrant's avatar
Nicolas Wavrant committed
847
  If not, download it in read-only mode"""
848
  default_git = os.path.join(config['runner_workdir'],
849
    'project', 'default_repo')
850 851 852
  if not os.path.exists(default_git):
    data = {'path': default_git,
            'repo': config['default_repo'],
Nicolas Wavrant's avatar
Nicolas Wavrant committed
853 854
    }
    cloneRepo(data)
855

856

857 858 859 860
def buildAndRun(config):
  runSoftwareWithLock(config)
  runInstanceWithLock(config)

861

862
def runSlapgridUntilSuccess(config, step):
863
  """Run slapos several times,
864
  in the maximum of the constant MAX_RUN_~~~~"""
865
  params = getBuildAndRunParams(config)
866
  if step == "instance":
867
    max_tries = (params['max_run_instance'] if params['run_instance'] else 0)
868 869
    runSlapgridWithLock = runInstanceWithLock
  elif step == "software":
870
    max_tries = (params['max_run_software'] if params['run_software'] else 0)
871 872
    runSlapgridWithLock = runSoftwareWithLock
  else:
873
    return -1
874 875
  counter_file = os.path.join(config['runner_workdir'], '.turn-left')
  open(counter_file, 'w+').write(str(max_tries))
876
  counter = max_tries
877
  slapgrid = True
878
  # XXX-Nico runSoftwareWithLock can return 0 or False (0==False)
879 880
  while counter > 0:
    counter -= 1
881 882 883
    slapgrid = runSlapgridWithLock(config, lock=True)
    # slapgrid == 0 because EXIT_SUCCESS == 0
    if slapgrid == 0:
884
      break
885 886 887 888 889 890
    times_left = int(open(counter_file).read()) - 1
    if times_left > 0 :
      open(counter_file, 'w+').write(str(times_left))
      counter = times_left
    else :
      counter = 0
891
  max_tries -= counter
892 893
  # run instance only if we are deploying the software release,
  # if it is defined so, and sr is correctly deployed
894
  if step == "software" and params['run_instance'] and slapgrid == 0:
895 896 897
    return (max_tries, runSlapgridUntilSuccess(config, "instance"))
  else:
    return max_tries
898 899


900 901
def setupDefaultSR(config):
  """If a default_sr is in the parameters,
902 903
  and no SR is deployed yet, setup it
  also run SR and Instance if required"""
904 905 906
  project = os.path.join(config['etc_dir'], '.project')
  if not os.path.exists(project) and config['default_sr'] != '':
    configNewSR(config, config['default_sr'])
907
  if config['auto_deploy']:
908
    thread.start_new_thread(buildAndRun, (config,))
909 910 911 912 913 914 915 916 917 918 919 920 921 922


def setMiniShellHistory(config, command):
  history_max_size = 10
  command = command + "\n"
  history_file = config['minishell_history_file']
  if os.path.exists(history_file):
    history = open(history_file, 'r').readlines()
    if len(history) >= history_max_size:
      del history[0]
  else:
    history = []
  history.append(command)
  open(history_file, 'w+').write(''.join(history))