utils.py 29.3 KB
Newer Older
1
# -*- coding: utf-8 -*-
Marco Mariani's avatar
Marco Mariani committed
2
# vim: set et sts=2:
Marco Mariani's avatar
Marco Mariani committed
3
# pylint: disable-msg=W0311,C0301,C0103,C0111,W0141,W0142
4

5
import ConfigParser
6
import datetime
7
import json
Marco Mariani's avatar
Marco Mariani committed
8
import logging
Marco Mariani's avatar
Marco Mariani committed
9
import md5
10
import os
11
import sup_process
12 13
import re
import shutil
14
import stat
15
import thread
Marco Mariani's avatar
Marco Mariani committed
16 17
import time
import urllib
18
import xmlrpclib
Alain Takoudjou's avatar
Alain Takoudjou committed
19
from xml.dom import minidom
Marco Mariani's avatar
Marco Mariani committed
20 21 22 23

import xml_marshaller
from flask import jsonify

24
from slapos.runner.gittools import cloneRepo
25

Nicolas Wavrant's avatar
Nicolas Wavrant committed
26
from slapos.runner.process import Popen
27 28
# from slapos.htpasswd import HtpasswdFile
from passlib.apache import HtpasswdFile
Marco Mariani's avatar
Marco Mariani committed
29
import slapos.slap
30
from slapos.grid.utils import md5digest
Łukasz Nowak's avatar
Łukasz Nowak committed
31

32
# Setup default flask (werkzeug) parser
Marco Mariani's avatar
Marco Mariani committed
33

34
logger = logging.getLogger('werkzeug')
Łukasz Nowak's avatar
Łukasz Nowak committed
35

36
TRUE_VALUES = (1, '1', True, 'true', 'True')
37

38 39 40 41 42 43 44
html_escape_table = {
  "&": "&",
  '"': """,
  "'": "'",
  ">": ">",
  "<": "&lt;",
}
45

46 47 48 49
def getBuildAndRunParams(config):
  json_file = os.path.join(config['etc_dir'], 'config.json')
  json_params = json.load(open(json_file))
  return json_params
Marco Mariani's avatar
Marco Mariani committed
50

51

52
def saveBuildAndRunParams(config, params):
53 54 55
  """XXX-Nico parameters have to be correct.
  Works like that because this function do not care
  about how you got the parameters"""
56 57 58
  json_file = os.path.join(config['etc_dir'], 'config.json')
  open(json_file, "w").write(json.dumps(params))

59

60 61
def html_escape(text):
  """Produce entities within text."""
Marco Mariani's avatar
Marco Mariani committed
62
  return "".join(html_escape_table.get(c, c) for c in text)
63

64 65 66 67
def getSession(config):
  """
  Get the session data of current user.
  Returns:
68
    a list of user information or None if the file does not exist.
69
  """
70
  user_path = os.path.join(config['etc_dir'], '.htpasswd')
71
  if os.path.exists(user_path):
72
    return open(user_path).read().split(';')
73

74 75 76 77 78 79
def checkUserCredential(config, username, password):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
  if not os.path.exists(htpasswdfile):
    return False
  passwd = HtpasswdFile(htpasswdfile)
  return passwd.check_password(username, password)
Marco Mariani's avatar
Marco Mariani committed
80

81
def updateUserCredential(config, username, password):
82 83 84 85
  """
  Save account information for the current user

  """
86 87 88 89
  if username and password:
    htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
    passwd = HtpasswdFile(htpasswdfile)
    passwd.set_password(username, password)
90
    passwd.save()
91
    return True
92 93

  return False
Łukasz Nowak's avatar
Łukasz Nowak committed
94

Marco Mariani's avatar
Marco Mariani committed
95

96 97 98 99 100 101 102 103
def getRcode(config):
  parser = ConfigParser.ConfigParser()
  try:
    parser.read(config['knowledge0_cfg'])
    return parser.get('public', 'recovery-code')
  except (ConfigParser.NoSectionError, IOError) as e:
    return None

104 105 106 107 108 109 110
def getUsernameList(config):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
  if os.path.exists(htpasswdfile):
    passwd = HtpasswdFile(htpasswdfile)
    return passwd.users()

  return []
111

112 113
def createNewUser(config, name, passwd):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
114 115
  try:
    htpasswd = HtpasswdFile(htpasswdfile, new=(not os.path.exists(htpasswdfile)))
116
    htpasswd.set_password(name, passwd)
117
    htpasswd.save()
118 119 120
  except IOError:
    return False
  return True
121

122 123 124 125 126 127
def getCurrentSoftwareReleaseProfile(config):
  """
  Returns used Software Release profile as a string.
  """
  try:
    software_folder = open(
128
        os.path.join(config['etc_dir'], ".project")).read().rstrip()
129 130
    return realpath(
        config, os.path.join(software_folder, config['software_profile']))
131
  # XXXX No Comments
132
  except IOError:
133
    return ''
134

Marco Mariani's avatar
Marco Mariani committed
135

136 137 138 139
def requestInstance(config, software_type=None):
  """
  Request the main instance of our environment
  """
140
  software_type_path = os.path.join(config['etc_dir'], ".software_type.xml")
141
  if software_type:
142 143 144
    # Write it to conf file for later use
    open(software_type_path, 'w').write(software_type)
  elif os.path.exists(software_type_path):
145
    software_type = open(software_type_path).read().rstrip()
146 147
  else:
    software_type = 'default'
148 149 150 151 152 153 154 155

  slap = slapos.slap.slap()
  profile = getCurrentSoftwareReleaseProfile(config)
  slap.initializeConnection(config['master_url'])

  param_path = os.path.join(config['etc_dir'], ".parameter.xml")
  xml_result = readParameters(param_path)
  partition_parameter_kw = None
Marco Mariani's avatar
Marco Mariani committed
156
  if type(xml_result) != type('') and 'instance' in xml_result:
157 158 159 160 161 162
    partition_parameter_kw = xml_result['instance']

  return slap.registerOpenOrder().request(
      profile,
      partition_reference=getSoftwareReleaseName(config),
      partition_parameter_kw=partition_parameter_kw,
163
      software_type=software_type,
164 165 166 167
      filter_kw=None,
      state=None,
      shared=False)

Marco Mariani's avatar
Marco Mariani committed
168

169
def updateProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
170 171 172 173
  """
  Configure Slapos Node computer and partitions.
  Send current Software Release to Slapproxy for compilation and deployment.
  """
174
  startProxy(config)
Łukasz Nowak's avatar
Łukasz Nowak committed
175 176 177
  if not os.path.exists(config['instance_root']):
    os.mkdir(config['instance_root'])
  slap = slapos.slap.slap()
178
  profile = getCurrentSoftwareReleaseProfile(config)
179

Łukasz Nowak's avatar
Łukasz Nowak committed
180
  slap.initializeConnection(config['master_url'])
181
  slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
Łukasz Nowak's avatar
Łukasz Nowak committed
182 183 184
  computer = slap.registerComputer(config['computer_id'])
  prefix = 'slappart'
  slap_config = {
Marco Mariani's avatar
Marco Mariani committed
185 186 187 188 189 190 191 192
    'address': config['ipv4_address'],
    'instance_root': config['instance_root'],
    'netmask': '255.255.255.255',
    'partition_list': [],
    'reference': config['computer_id'],
    'software_root': config['software_root']
  }

Łukasz Nowak's avatar
Łukasz Nowak committed
193 194 195 196 197 198
  for i in xrange(0, int(config['partition_amount'])):
    partition_reference = '%s%s' % (prefix, i)
    partition_path = os.path.join(config['instance_root'], partition_reference)
    if not os.path.exists(partition_path):
      os.mkdir(partition_path)
    os.chmod(partition_path, 0750)
Marco Mariani's avatar
Marco Mariani committed
199 200 201 202 203 204 205 206 207 208 209 210 211
    slap_config['partition_list'].append({
                                           'address_list': [
                                              {
                                                'addr': config['ipv4_address'],
                                                'netmask': '255.255.255.255'
                                              }, {
                                                'addr': config['ipv6_address'],
                                                'netmask': 'ffff:ffff:ffff::'
                                              },
                                           ],
                                           'path': partition_path,
                                           'reference': partition_reference,
                                           'tap': {'name': partition_reference}})
Marco Mariani's avatar
Marco Mariani committed
212
  computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
213
  return True
Łukasz Nowak's avatar
Łukasz Nowak committed
214

215

216
def updateInstanceParameter(config, software_type=None):
Alain Takoudjou's avatar
Alain Takoudjou committed
217 218 219 220 221 222 223
  """
  Reconfigure Slapproxy to re-deploy current Software Instance with parameters.

  Args:
    config: Slaprunner configuration.
    software_type: reconfigure Software Instance with software type.
  """
224
  time.sleep(1)
225
  if not (updateProxy(config) and requestInstance(config, software_type)):
226 227
    return False

228

Łukasz Nowak's avatar
Łukasz Nowak committed
229
def startProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
230
  """Start Slapproxy server"""
231
  if sup_process.isRunning(config, 'slapproxy'):
232
    return
233
  try:
234
    return sup_process.runProcess(config, "slapproxy")
235 236
  except xmlrpclib.Fault:
    pass
237
  time.sleep(4)
Łukasz Nowak's avatar
Łukasz Nowak committed
238 239 240


def stopProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
241
  """Stop Slapproxy server"""
242
  return sup_process.stopProcess(config, "slapproxy")
Łukasz Nowak's avatar
Łukasz Nowak committed
243 244 245


def removeProxyDb(config):
246
  """Remove Slapproxy database, this is used to initialize proxy for example when
Alain Takoudjou's avatar
Alain Takoudjou committed
247
    configuring new Software Release"""
Łukasz Nowak's avatar
Łukasz Nowak committed
248 249 250
  if os.path.exists(config['database_uri']):
    os.unlink(config['database_uri'])

Marco Mariani's avatar
Marco Mariani committed
251

252
def isSoftwareRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
253
  """
254
    Return True if slapos is still running and false if slapos if not
Alain Takoudjou's avatar
Alain Takoudjou committed
255
  """
256
  return sup_process.isRunning(config, 'slapgrid-sr')
Łukasz Nowak's avatar
Łukasz Nowak committed
257 258


259 260 261 262 263 264 265
def slapgridResultToFile(config, step, returncode, datetime):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  result = {'last_build':datetime, 'success':returncode}
  open(file, "w").write(json.dumps(result))


266 267 268 269 270 271 272 273 274 275
def getSlapgridResult(config, step):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  if os.path.exists(file):
    result = json.loads(open(file, "r").read())
  else:
    result = {'last_build': 0, 'success':-1}
  return result


276 277 278 279 280
def waitProcess(config, process, step):
  process.wait()
  date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  slapgridResultToFile(config, step, process.returncode, date)

281
def runSlapgridWithLock(config, step, process_name, lock=False):
Alain Takoudjou's avatar
Alain Takoudjou committed
282
  """
283 284 285 286
  * process_name is the name of the process given to supervisord, which will
    run the software or the instance
  * step is one of ('software', 'instance')
  * lock allows to make this function asynchronous or not
Alain Takoudjou's avatar
Alain Takoudjou committed
287
  """
288
  if sup_process.isRunning(config, process_name):
289
    return 1
290

291 292 293 294 295 296
  root_folder = config["%s_root" % step]
  log_file = config["%s_log" % step]

  if not os.path.exists(root_folder):
    os.mkdir(root_folder)

297
  # XXX Hackish and unreliable
298 299
  if os.path.exists(log_file):
    os.remove(log_file)
300
  if not updateProxy(config):
301
    return 1
302 303
  if step == 'instance' and not requestInstance(config):
    return 1
304
  try:
305
    sup_process.runProcess(config, process_name)
306
    if lock:
307
      sup_process.waitForProcessEnd(config, process_name)
308
    #Saves the current compile software for re-use
309 310 311
    if step == 'software':
      config_SR_folder(config)
    return sup_process.returnCode(config, process_name)
312
  except xmlrpclib.Fault:
313
    return 1
314

Łukasz Nowak's avatar
Łukasz Nowak committed
315

316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
def runSoftwareWithLock(config, lock=False):
  """
    Use Slapgrid to compile current Software Release and wait until
    compilation is done
  """
  return runSlapgridWithLock(config, 'software', 'slapgrid-sr', lock)


def runInstanceWithLock(config, lock=False):
  """
    Use Slapgrid to deploy current Software Release and wait until
    deployment is done.
  """
  return runSlapgridWithLock(config, 'instance', 'slapgrid-cp', lock)


332
def config_SR_folder(config):
Marco Mariani's avatar
Marco Mariani committed
333 334
  """Create a symbolik link for each folder in software folder. That allows
    the user to customize software release folder"""
335
  config_name = 'slaprunner.config'
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
  def link_to_folder(name, folder):
    destination = os.path.join(config['software_link'], name)
    source = os.path.join(config['software_root'], folder)
    cfg = os.path.join(destination, config_name)
        #create symlink
    if os.path.lexists(destination):
      os.remove(destination)
    os.symlink(source, destination)
        #write config file
    if os.path.exists(source):
      with open(cfg, 'w') as cf:
        cf.write(current_project + '#' + folder)

  # First create the link for current project
  current_project = open(os.path.join(config['etc_dir'], ".project")).read()
  profile = getCurrentSoftwareReleaseProfile(config)
  if current_project[-1] == '/':
     current_project = current_project[:-1]
  name = current_project.split('/')[-1]
  md5sum = md5digest(profile)
  link_to_folder(name, md5sum)
  # check other links
  # XXX-Marco do not shadow 'list'
  list = []
360 361 362
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
363
      cfg = open(cfg_path).read().split("#")
364
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
365
        continue  # there is a broken config file
366
      list.append(cfg[1])
367 368 369 370
  if os.path.exists(config['software_root']):
    folder_list = os.listdir(config['software_root'])
  else:
    return
371
  if not folder_list:
372 373 374
    return
  for folder in folder_list:
    if folder in list:
Marco Mariani's avatar
Marco Mariani committed
375
      continue  # this folder is already registered
376
    else:
377
      link_to_folder(folder, folder)
Marco Mariani's avatar
Marco Mariani committed
378

379 380 381 382 383 384 385
def loadSoftwareRList(config):
  """Return list (of dict) of Software Release from symbolik SR folder"""
  list = []
  config_name = 'slaprunner.config'
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
386
      cfg = open(cfg_path).read().split("#")
387
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
388
        continue  # there is a broken config file
389 390
      list.append(dict(md5=cfg[1], path=cfg[0], title=path))
  return list
Łukasz Nowak's avatar
Łukasz Nowak committed
391

Marco Mariani's avatar
Marco Mariani committed
392

393
def isInstanceRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
394
  """
395
    Return True if slapos is still running and False otherwise
Alain Takoudjou's avatar
Alain Takoudjou committed
396
  """
397
  return sup_process.isRunning(config, 'slapgrid-cp')
Łukasz Nowak's avatar
Łukasz Nowak committed
398 399


Alain Takoudjou's avatar
Alain Takoudjou committed
400 401 402 403 404 405 406 407 408 409 410 411
def getProfilePath(projectDir, profile):
  """
  Return the path of the current Software Release `profile`

  Args:
    projectDir: Slaprunner workspace location.
    profile: file to search into the workspace.

  Returns:
    String, path of current Software Release profile
  """
  if not os.path.exists(os.path.join(projectDir, ".project")):
412
    return False
Alain Takoudjou's avatar
Alain Takoudjou committed
413
  projectFolder = open(os.path.join(projectDir, ".project")).read()
414
  return os.path.join(projectFolder, profile)
Łukasz Nowak's avatar
Łukasz Nowak committed
415

Marco Mariani's avatar
Marco Mariani committed
416

Łukasz Nowak's avatar
Łukasz Nowak committed
417
def getSlapStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
418
  """Return all Slapos Partitions with associate information"""
Łukasz Nowak's avatar
Łukasz Nowak committed
419 420 421 422 423 424 425
  slap = slapos.slap.slap()
  slap.initializeConnection(config['master_url'])
  partition_list = []
  computer = slap.registerComputer(config['computer_id'])
  try:
    for partition in computer.getComputerPartitionList():
      # Note: Internal use of API, as there is no reflexion interface in SLAP
426
      partition_list.append((partition.getId(), partition._connection_dict.copy()))
Łukasz Nowak's avatar
Łukasz Nowak committed
427 428
  except Exception:
    pass
429 430 431
  if partition_list:
    for i in xrange(0, int(config['partition_amount'])):
      slappart_id = '%s%s' % ("slappart", i)
Marco Mariani's avatar
Marco Mariani committed
432
      if not [x[0] for x in partition_list if slappart_id == x[0]]:
433
        partition_list.append((slappart_id, []))
Łukasz Nowak's avatar
Łukasz Nowak committed
434 435
  return partition_list

Marco Mariani's avatar
Marco Mariani committed
436

Łukasz Nowak's avatar
Łukasz Nowak committed
437
def svcStopAll(config):
438
  """Stop all Instance processes on this computer"""
439
  try:
440
    return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
441
                  'stop', 'all']).communicate()[0]
442 443
  except:
    pass
Łukasz Nowak's avatar
Łukasz Nowak committed
444

445 446 447 448 449 450 451
def svcStartAll(config):
  """Start all Instance processes on this computer"""
  try:
    return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
                  'start', 'all']).communicate()[0]
  except:
    pass
Marco Mariani's avatar
Marco Mariani committed
452

453 454
def removeInstanceRootDirectory(config):
  """Clean instance directory"""
455
  if os.path.exists(config['instance_root']):
456 457 458 459 460 461 462 463 464 465 466 467 468
    for instance_directory in os.listdir(config['instance_root']):
      instance_directory = os.path.join(config['instance_root'], instance_directory)
      # XXX: hardcoded
      if stat.S_ISSOCK(os.stat(instance_directory).st_mode) or os.path.isfile(instance_directory):
        # Ignore non-instance related files
        continue
      for root, dirs, _ in os.walk(instance_directory):
        for fname in dirs:
          fullPath = os.path.join(root, fname)
          if not os.access(fullPath, os.W_OK):
            # Some directories may be read-only, preventing to remove files in it
            os.chmod(fullPath, 0744)
      shutil.rmtree(instance_directory)
469

470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
def removeCurrentInstance(config):
  if isInstanceRunning(config):
    return "Instantiation in progress, cannot remove instance"

  # Stop all processes
  svcStopAll(config)
  if stopProxy(config):
    removeProxyDb(config)
  else:
    return "Something went wrong when trying to stop slapproxy."

  # Remove Instance directory and data related to the instance
  try:
    removeInstanceRootDirectory(config)
    param_path = os.path.join(config['etc_dir'], ".parameter.xml")
    if os.path.exists(param_path):
      os.remove(param_path)
  except IOError:
    return "The filesystem couldn't been cleaned properly"
  return True

Marco Mariani's avatar
Marco Mariani committed
491

Łukasz Nowak's avatar
Łukasz Nowak committed
492
def getSvcStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
493
  """Return all Softwares Instances process Information"""
494
  result = Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
495
                  'status']).communicate()[0]
496
  regex = "(^unix:.+\.socket)|(^error:)|(^watchdog).*$"
497 498
  supervisord = []
  for item in result.split('\n'):
499
    if item.strip() != "":
Marco Mariani's avatar
Marco Mariani committed
500
      if re.search(regex, item, re.IGNORECASE) is None:
501
        supervisord.append(re.split('[\s,]+', item))
502 503
  return supervisord

Marco Mariani's avatar
Marco Mariani committed
504

505
def getSvcTailProcess(config, process):
506
  """Get log for the specified process
Alain Takoudjou's avatar
Alain Takoudjou committed
507 508 509

  Args:
    config: Slaprunner configuration
510
    process: process name. this value is passed to supervisord.
Alain Takoudjou's avatar
Alain Takoudjou committed
511 512 513
  Returns:
    a string that contains the log of the process.
  """
514
  return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
515 516
                "tail", process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
517

518
def svcStartStopProcess(config, process, action):
Alain Takoudjou's avatar
Alain Takoudjou committed
519 520 521 522 523 524 525
  """Send start or stop process command to supervisord

  Args:
    config: Slaprunner configuration.
    process: process to start or stop.
    action: current state which is used to generate the new process state.
  """
Marco Mariani's avatar
Marco Mariani committed
526 527 528 529 530 531 532
  cmd = {
    'RESTART': 'restart',
    'STOPPED': 'start',
    'RUNNING': 'stop',
    'EXITED': 'start',
    'STOP': 'stop'
  }
533
  return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
534 535
                cmd[action], process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
536

537 538
def listFolder(config, path):
  """Return the list of folder into path
Alain Takoudjou's avatar
Alain Takoudjou committed
539 540

  Agrs:
541
    path: path of the directory to list
Alain Takoudjou's avatar
Alain Takoudjou committed
542 543 544
  Returns:
    a list that contains each folder name.
  """
545 546 547 548 549 550 551 552
  folderList = []
  folder = realpath(config, path)
  if folder:
    path_list = sorted(os.listdir(folder), key=str.lower)
    for elt in path_list:
      if os.path.isdir(os.path.join(folder, elt)):
        folderList.append(elt)
  return folderList
553

Marco Mariani's avatar
Marco Mariani committed
554

555
def configNewSR(config, projectpath):
Alain Takoudjou's avatar
Alain Takoudjou committed
556 557 558 559 560 561 562 563
  """Configure a Software Release as current Software Release

  Args:
    config: slaprunner configuration
    projectpath: path of the directory that contains the software realease to configure
  Returns:
    True if all is done well, otherwise return false.
  """
564 565
  folder = realpath(config, projectpath)
  if folder:
566 567
    sup_process.stopProcess(config, 'slapgrid-cp')
    sup_process.stopProcess(config, 'slapgrid-sr')
568
    removeCurrentInstance(config)
569
    open(os.path.join(config['etc_dir'], ".project"), 'w').write(projectpath)
570 571 572 573
    return True
  else:
    return False

Marco Mariani's avatar
Marco Mariani committed
574

575
def newSoftware(folder, config, session):
Alain Takoudjou's avatar
Alain Takoudjou committed
576 577 578 579 580 581 582
  """
  Create a new Software Release folder with default profiles

  Args:
    folder: directory of the new software release
    config: slraprunner configuration
    session: Flask session directory"""
583 584
  json = ""
  code = 0
585
  basedir = config['etc_dir']
586
  try:
587 588 589
    folderPath = realpath(config, folder, check_exist=False)
    if folderPath and not os.path.exists(folderPath):
      os.mkdir(folderPath)
590 591 592 593
      #load software.cfg and instance.cfg from http://git.erp5.org
      software = "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/lamp-template/software.cfg"
      softwareContent = ""
      try:
594
        softwareContent = urllib.urlopen(software).read()
595
      except:
596 597
        #Software.cfg and instance.cfg content will be empty
        pass
598
      open(os.path.join(folderPath, config['software_profile']), 'w').write(softwareContent)
599 600
      open(os.path.join(folderPath, config['instance_profile']), 'w').write("")
      open(os.path.join(basedir, ".project"), 'w').write(folder + "/")
601 602 603 604 605 606
      #Clean sapproxy Database
      stopProxy(config)
      removeProxyDb(config)
      startProxy(config)
      #Stop runngin process and remove existing instance
      removeInstanceRoot(config)
607 608 609
      session['title'] = getProjectTitle(config)
      code = 1
    else:
Marco Mariani's avatar
Marco Mariani committed
610
      json = "Bad folder or Directory '%s' already exist, please enter a new name for your software" % folder
Marco Mariani's avatar
Marco Mariani committed
611
  except Exception as e:
Marco Mariani's avatar
Marco Mariani committed
612
    json = "Can not create your software, please try again! : %s " % e
613 614
    if os.path.exists(folderPath):
      shutil.rmtree(folderPath)
615 616
  return jsonify(code=code, result=json)

Marco Mariani's avatar
Marco Mariani committed
617

618
def checkSoftwareFolder(path, config):
Alain Takoudjou's avatar
Alain Takoudjou committed
619
  """Check id `path` is a valid Software Release folder"""
620 621
  realdir = realpath(config, path)
  if realdir and os.path.exists(os.path.join(realdir, config['software_profile'])):
622 623 624
    return jsonify(result=path)
  return jsonify(result="")

Marco Mariani's avatar
Marco Mariani committed
625

626
def getProjectTitle(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
627
  """Generate the name of the current software Release (for slaprunner UI)"""
628
  conf = os.path.join(config['etc_dir'], ".project")
629 630 631 632
  # instance_name is optional parameter
  instance_name = config.get('instance_name')
  if instance_name:
    instance_name = '%s - ' % instance_name
633
  if os.path.exists(conf):
634
    project = open(conf, "r").read().split("/")
Marco Mariani's avatar
Marco Mariani committed
635
    software = project[-2]
636 637
    return '%s%s (%s)' % (instance_name, software, '/'.join(project[:-2]))
  return "%sNo Profile" % instance_name
638

Marco Mariani's avatar
Marco Mariani committed
639

640
def getSoftwareReleaseName(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
641
  """Get the name of the current Software Release"""
642
  sr_profile = os.path.join(config['etc_dir'], ".project")
643 644
  if os.path.exists(sr_profile):
    project = open(sr_profile, "r").read().split("/")
Marco Mariani's avatar
Marco Mariani committed
645
    software = project[-2]
646 647 648
    return software.replace(' ', '_')
  return "No_name"

649 650 651
def removeSoftwareRootDirectory(config, md5, folder_name):
  """
  Removes all content in the filesystem of the software release specified by md5
Alain Takoudjou's avatar
Alain Takoudjou committed
652 653 654

  Args:
    config: slaprunner configuration
655 656 657
    folder_name: the link name given to the software release
    md5: the md5 filename given by slapgrid to SR folder
  """
658
  path = os.path.join(config['software_root'], md5)
659
  linkpath = os.path.join(config['software_link'], folder_name)
660
  if not os.path.exists(path):
661
    return (0, "Cannot remove software Release: No such file or directory")
662
  if not os.path.exists(linkpath):
663 664
    return (0, "Cannot remove software Release: No such file or directory %s" %
                    ('software_root/' + folder_name))
665
  os.unlink(linkpath)
666
  shutil.rmtree(path)
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
  return

def removeSoftwareByName(config, md5, folder_name):
  """
  Removes a software release specified by its md5 and its name from the webrunner.
  If the software release is the one of the current running instance, then
  the instance should be stopped.

  Args:
    config: slaprunner configuration
    folder_name: the link name given to the software release
    md5: the md5 filename given by slapgrid to SR folder
  """
  if isSoftwareRunning(config) or isInstanceRunning(config):
    return (0, "Software installation or instantiation in progress, cannot remove")

  if getSoftwareReleaseName(config) == folder_name:
    removeCurrentInstance(config)

  result = removeSoftwareRootDirectory(config, md5, folder_name)
  if result is not None:
    return result
  return 1, loadSoftwareRList(config)
690

Marco Mariani's avatar
Marco Mariani committed
691

692 693 694 695 696 697 698 699 700 701 702 703
def tail(f, lines=20):
  """
  Returns the last `lines` lines of file `f`. It is an implementation of tail -f n.
  """
  BUFSIZ = 1024
  f.seek(0, 2)
  bytes = f.tell()
  size = lines + 1
  block = -1
  data = []
  while size > 0 and bytes > 0:
      if bytes - BUFSIZ > 0:
704 705 706 707
          # Seek back one whole BUFSIZ
          f.seek(block * BUFSIZ, 2)
          # read BUFFER
          data.insert(0, f.read(BUFSIZ))
708
      else:
709
          # file too small, start from begining
Marco Mariani's avatar
Marco Mariani committed
710
          f.seek(0, 0)
711 712
          # only read what was not read
          data.insert(0, f.read(bytes))
713 714 715 716
      linesFound = data[0].count('\n')
      size -= linesFound
      bytes -= BUFSIZ
      block -= 1
Marco Mariani's avatar
Marco Mariani committed
717
  return '\n'.join(''.join(data).splitlines()[-lines:])
718

Marco Mariani's avatar
Marco Mariani committed
719

720
def readFileFrom(f, lastPosition, limit=20000):
721 722 723
  """
  Returns the last lines of file `f`, from position lastPosition.
  and the last position
724
  limit = max number of characters to read
725 726 727
  """
  BUFSIZ = 1024
  f.seek(0, 2)
Marco Mariani's avatar
Marco Mariani committed
728
  # XXX-Marco do now shadow 'bytes'
729 730 731 732
  bytes = f.tell()
  block = -1
  data = ""
  length = bytes
Marco Mariani's avatar
Marco Mariani committed
733 734
  truncated = False  # True if a part of log data has been truncated
  if (lastPosition <= 0 and length > limit) or (length - lastPosition > limit):
735
    lastPosition = length - limit
736
    truncated = True
737 738
  size = bytes - lastPosition
  while bytes > lastPosition:
Marco Mariani's avatar
Marco Mariani committed
739
    if abs(block * BUFSIZ) <= size:
740 741 742 743
      # Seek back one whole BUFSIZ
      f.seek(block * BUFSIZ, 2)
      data = f.read(BUFSIZ) + data
    else:
Marco Mariani's avatar
Marco Mariani committed
744
      margin = abs(block * BUFSIZ) - size
745
      if length < BUFSIZ:
Marco Mariani's avatar
Marco Mariani committed
746
        f.seek(0, 0)
747
      else:
748 749
        seek = block * BUFSIZ + margin
        f.seek(seek, 2)
750 751 752 753
      data = f.read(BUFSIZ - margin) + data
    bytes -= BUFSIZ
    block -= 1
  f.close()
Marco Mariani's avatar
Marco Mariani committed
754 755 756 757 758 759
  return {
    'content': data,
    'position': length,
    'truncated': truncated
  }

760

761 762 763 764
def isText(file):
  """Return True if the mimetype of file is Text"""
  if not os.path.exists(file):
    return False
Marco Mariani's avatar
Marco Mariani committed
765
  text_range = ''.join(map(chr, [7, 8, 9, 10, 12, 13, 27] + range(0x20, 0x100)))
766 767 768
  is_binary_string = lambda bytes: bool(bytes.translate(None, text_range))
  try:
    return not is_binary_string(open(file).read(1024))
769 770
  except:
    return False
771

Marco Mariani's avatar
Marco Mariani committed
772

773 774
def md5sum(file):
  """Compute md5sum of `file` and return hexdigest value"""
775
  # XXX-Marco: returning object or False boolean is an anti-pattern. better to return object or None
776 777 778 779
  if os.path.isdir(file):
    return False
  try:
    fh = open(file, 'rb')
Marco Mariani's avatar
Marco Mariani committed
780
    m = md5.md5()
781 782 783
    while True:
      data = fh.read(8192)
      if not data:
784
        break
785 786
      m.update(data)
    return m.hexdigest()
787
  except:
788 789
    return False

Marco Mariani's avatar
Marco Mariani committed
790

791
def realpath(config, path, check_exist=True):
792 793 794 795
  """
  Get realpath of path or return False if user is not allowed to access to
  this file.
  """
796 797
  split_path = path.split('/')
  key = split_path[0]
798 799 800
  virtual_path_list = ('software_root', 'instance_root', 'workspace',
    'runner_workdir', 'software_link')
  if key not in virtual_path_list:
801
    return ''
802
  allow_list = {path: config[path] for path in virtual_path_list if path in config}
Marco Mariani's avatar
Marco Mariani committed
803 804 805 806
  del split_path[0]
  path = os.path.join(allow_list[key], *split_path)
  if check_exist:
    if os.path.exists(path):
807
      return path
Marco Mariani's avatar
Marco Mariani committed
808
    else:
809
      return ''
Marco Mariani's avatar
Marco Mariani committed
810 811 812
  else:
    return path

813 814

def readParameters(path):
Alain Takoudjou's avatar
Alain Takoudjou committed
815 816 817 818 819 820
  """Read Instance parameters stored into a local file.

  Agrs:
    path: path of the xml file that contains parameters

  Return:
Marco Mariani's avatar
Marco Mariani committed
821
    a dictionary of instance parameters."""
822 823
  if os.path.exists(path):
    try:
Alain Takoudjou's avatar
Alain Takoudjou committed
824
      xmldoc = minidom.parse(path)
Alain Takoudjou's avatar
Alain Takoudjou committed
825
      obj = {}
826
      for elt in xmldoc.childNodes:
Marco Mariani's avatar
Marco Mariani committed
827
        sub_obj = {}
828 829
        for subnode in elt.childNodes:
          if subnode.nodeType != subnode.TEXT_NODE:
Marco Mariani's avatar
Marco Mariani committed
830
            sub_obj[str(subnode.getAttribute('id'))] = subnode.childNodes[0].data  # .decode('utf-8').decode('utf-8')
Marco Mariani's avatar
Marco Mariani committed
831 832
            obj[str(elt.tagName)] = sub_obj
      return obj
833 834 835
    except Exception, e:
      return str(e)
  else:
Marco Mariani's avatar
Marco Mariani committed
836
    return "No such file or directory: %s" % path
Nicolas Wavrant's avatar
Nicolas Wavrant committed
837

838

839 840 841 842
def isSoftwareReleaseReady(config):
  """Return 1 if the Software Release has
  correctly been deployed, 0 if not,
  and 2 if it is currently deploying"""
843 844
  auto_deploy = config['auto_deploy'] in TRUE_VALUES
  auto_run = config['autorun'] in TRUE_VALUES
845
  project = os.path.join(config['etc_dir'], '.project')
846
  if not ( os.path.exists(project) and (auto_run or auto_deploy) ):
847
    return "0"
848
  path = open(project, 'r').readline().strip()
849 850 851 852
  software_name = path
  if software_name[-1] == '/':
    software_name = software_name[:-1]
  software_name = software_name.split('/')[-1]
853
  updateInstanceParameter(config)
854
  config_SR_folder(config)
855
  if os.path.exists(os.path.join(config['runner_workdir'],
856
      'softwareLink', software_name, '.completed')):
857
    if auto_run:
858
      runSlapgridUntilSuccess(config, 'instance')
859 860 861 862
    return "1"
  else:
    if isSoftwareRunning(config):
      return "2"
863
    elif auto_deploy:
864
      runSoftwareWithLock(config)
865 866
      config_SR_folder(config)
      time.sleep(15)
867
      if auto_run:
868
        runSlapgridUntilSuccess(config, 'instance')
869 870 871
      return "2"
    else:
      return "0"
Nicolas Wavrant's avatar
Nicolas Wavrant committed
872

873

874
def cloneDefaultGit(config):
875
  """Test if the default git has been downloaded yet
Nicolas Wavrant's avatar
Nicolas Wavrant committed
876
  If not, download it in read-only mode"""
877
  default_git = os.path.join(config['runner_workdir'],
878
    'project', 'default_repo')
879 880 881
  if not os.path.exists(default_git):
    data = {'path': default_git,
            'repo': config['default_repo'],
Nicolas Wavrant's avatar
Nicolas Wavrant committed
882 883
    }
    cloneRepo(data)
884

885

886 887 888 889
def buildAndRun(config):
  runSoftwareWithLock(config)
  runInstanceWithLock(config)

890

891
def runSlapgridUntilSuccess(config, step):
892
  """Run slapos several times,
893
  in the maximum of the constant MAX_RUN_~~~~"""
894
  params = getBuildAndRunParams(config)
895
  if step == "instance":
896
    max_tries = (params['max_run_instance'] if params['run_instance'] else 0)
897 898
    runSlapgridWithLock = runInstanceWithLock
  elif step == "software":
899
    max_tries = (params['max_run_software'] if params['run_software'] else 0)
900 901
    runSlapgridWithLock = runSoftwareWithLock
  else:
902
    return -1
903 904
  counter_file = os.path.join(config['runner_workdir'], '.turn-left')
  open(counter_file, 'w+').write(str(max_tries))
905
  counter = max_tries
906
  slapgrid = True
907
  # XXX-Nico runSoftwareWithLock can return 0 or False (0==False)
908 909
  while counter > 0:
    counter -= 1
910 911 912
    slapgrid = runSlapgridWithLock(config, lock=True)
    # slapgrid == 0 because EXIT_SUCCESS == 0
    if slapgrid == 0:
913
      break
914 915 916 917 918 919
    times_left = int(open(counter_file).read()) - 1
    if times_left > 0 :
      open(counter_file, 'w+').write(str(times_left))
      counter = times_left
    else :
      counter = 0
920
  max_tries -= counter
921 922
  # run instance only if we are deploying the software release,
  # if it is defined so, and sr is correctly deployed
923
  if step == "software" and params['run_instance'] and slapgrid == 0:
924 925 926
    return (max_tries, runSlapgridUntilSuccess(config, "instance"))
  else:
    return max_tries
927 928


929 930
def setupDefaultSR(config):
  """If a default_sr is in the parameters,
931 932
  and no SR is deployed yet, setup it
  also run SR and Instance if required"""
933 934 935
  project = os.path.join(config['etc_dir'], '.project')
  if not os.path.exists(project) and config['default_sr'] != '':
    configNewSR(config, config['default_sr'])
936
  if config['auto_deploy']:
937
    thread.start_new_thread(buildAndRun, (config,))
938 939 940 941 942 943 944 945 946 947 948 949 950 951


def setMiniShellHistory(config, command):
  history_max_size = 10
  command = command + "\n"
  history_file = config['minishell_history_file']
  if os.path.exists(history_file):
    history = open(history_file, 'r').readlines()
    if len(history) >= history_max_size:
      del history[0]
  else:
    history = []
  history.append(command)
  open(history_file, 'w+').write(''.join(history))