utils.py 29 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
# vim: set et sts=2:
Marco Mariani's avatar
Marco Mariani committed
3
# pylint: disable-msg=W0311,C0301,C0103,C0111,W0141,W0142
4

5
from six.moves import configparser
6
import datetime
7
import json
Marco Mariani's avatar
Marco Mariani committed
8
import logging
9
import hashlib
10
import os
11
from . import sup_process
12 13
import re
import shutil
14
import stat
15
from six.moves import _thread, range
Marco Mariani's avatar
Marco Mariani committed
16
import time
17 18
from six.moves.urllib.request import urlopen
import six.moves.xmlrpc_client as xmlrpclib
19
from xml.dom import minidom
Marco Mariani's avatar
Marco Mariani committed
20 21 22 23

import xml_marshaller
from flask import jsonify

24
from slapos.runner.gittools import cloneRepo
25

26
from slapos.runner.process import Popen
27 28
# from slapos.htpasswd import HtpasswdFile
from passlib.apache import HtpasswdFile
Marco Mariani's avatar
Marco Mariani committed
29
import slapos.slap
30
from slapos.grid.utils import md5digest
31
from slapos.util import str2bytes
Łukasz Nowak's avatar
Łukasz Nowak committed
32

33
logger = logging.getLogger('slaprunner')
Łukasz Nowak's avatar
Łukasz Nowak committed
34

35
TRUE_VALUES = (1, '1', True, 'true', 'True')
36

37 38 39 40 41 42 43
html_escape_table = {
  "&": "&",
  '"': """,
  "'": "'",
  ">": ">",
  "<": "&lt;",
}
44

45 46
def getBuildAndRunParams(config):
  json_file = os.path.join(config['etc_dir'], 'config.json')
47 48
  with open(json_file) as f:
    json_params = json.load(f)
49
  return json_params
Marco Mariani's avatar
Marco Mariani committed
50

51

52
def saveBuildAndRunParams(config, params):
53 54 55
  """XXX-Nico parameters have to be correct.
  Works like that because this function do not care
  about how you got the parameters"""
56
  json_file = os.path.join(config['etc_dir'], 'config.json')
57
  with open(json_file, "w") as f:
58
    json.dump(params, f)
59

60

61 62
def html_escape(text):
  """Produce entities within text."""
Marco Mariani's avatar
Marco Mariani committed
63
  return "".join(html_escape_table.get(c, c) for c in text)
64

65 66 67 68
def getSession(config):
  """
  Get the session data of current user.
  Returns:
69
    a list of user information or None if the file does not exist.
70
  """
71
  user_path = os.path.join(config['etc_dir'], '.htpasswd')
72
  if os.path.exists(user_path):
73 74
    with open(user_path) as f:
      return f.read().split(';')
75

76 77 78 79 80 81
def checkUserCredential(config, username, password):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
  if not os.path.exists(htpasswdfile):
    return False
  passwd = HtpasswdFile(htpasswdfile)
  return passwd.check_password(username, password)
Marco Mariani's avatar
Marco Mariani committed
82

83
def updateUserCredential(config, username, password):
84 85 86 87
  """
  Save account information for the current user

  """
88 89 90 91
  if username and password:
    htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
    passwd = HtpasswdFile(htpasswdfile)
    passwd.set_password(username, password)
92
    passwd.save()
93
    return True
94 95

  return False
Łukasz Nowak's avatar
Łukasz Nowak committed
96

Marco Mariani's avatar
Marco Mariani committed
97

98
def getRcode(config):
99
  parser = configparser.ConfigParser()
100 101 102
  try:
    parser.read(config['knowledge0_cfg'])
    return parser.get('public', 'recovery-code')
103
  except (configparser.NoSectionError, IOError) as e:
104 105
    return None

106 107 108 109 110 111 112
def getUsernameList(config):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
  if os.path.exists(htpasswdfile):
    passwd = HtpasswdFile(htpasswdfile)
    return passwd.users()

  return []
113

114 115
def createNewUser(config, name, passwd):
  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
116 117
  try:
    htpasswd = HtpasswdFile(htpasswdfile, new=(not os.path.exists(htpasswdfile)))
118
    htpasswd.set_password(name, passwd)
119
    htpasswd.save()
120 121 122
  except IOError:
    return False
  return True
123

124 125 126 127 128
def getCurrentSoftwareReleaseProfile(config):
  """
  Returns used Software Release profile as a string.
  """
  try:
129 130
    with open(os.path.join(config['etc_dir'], ".project")) as f:
        software_folder = f.read().rstrip()
131 132
    return realpath(
        config, os.path.join(software_folder, config['software_profile']))
133
  # XXXX No Comments
134
  except IOError:
135
    return ''
136

Marco Mariani's avatar
Marco Mariani committed
137

138 139 140 141
def requestInstance(config, software_type=None):
  """
  Request the main instance of our environment
  """
142
  software_type_path = os.path.join(config['etc_dir'], ".software_type.xml")
143
  if software_type:
144
    # Write it to conf file for later use
145 146
    with open(software_type_path, 'w') as f:
      f.write(software_type)
147
  elif os.path.exists(software_type_path):
148 149
    with open(software_type_path) as f:
      software_type = f.read().rstrip()
150 151
  else:
    software_type = 'default'
152 153 154 155 156 157 158 159

  slap = slapos.slap.slap()
  profile = getCurrentSoftwareReleaseProfile(config)
  slap.initializeConnection(config['master_url'])

  param_path = os.path.join(config['etc_dir'], ".parameter.xml")
  xml_result = readParameters(param_path)
  partition_parameter_kw = None
Marco Mariani's avatar
Marco Mariani committed
160
  if type(xml_result) != type('') and 'instance' in xml_result:
161 162 163 164 165 166
    partition_parameter_kw = xml_result['instance']

  return slap.registerOpenOrder().request(
      profile,
      partition_reference=getSoftwareReleaseName(config),
      partition_parameter_kw=partition_parameter_kw,
167
      software_type=software_type,
168 169 170 171
      filter_kw=None,
      state=None,
      shared=False)

Marco Mariani's avatar
Marco Mariani committed
172

173
def updateProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
174 175 176 177
  """
  Configure Slapos Node computer and partitions.
  Send current Software Release to Slapproxy for compilation and deployment.
  """
178
  startProxy(config)
Łukasz Nowak's avatar
Łukasz Nowak committed
179
  slap = slapos.slap.slap()
180
  profile = getCurrentSoftwareReleaseProfile(config)
Łukasz Nowak's avatar
Łukasz Nowak committed
181
  slap.initializeConnection(config['master_url'])
182
  slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
183 184
  
  runFormatWithLock(config, lock=True)
185
  return True
Łukasz Nowak's avatar
Łukasz Nowak committed
186

187

188
def updateInstanceParameter(config, software_type=None):
Alain Takoudjou's avatar
Alain Takoudjou committed
189 190 191 192 193 194 195
  """
  Reconfigure Slapproxy to re-deploy current Software Instance with parameters.

  Args:
    config: Slaprunner configuration.
    software_type: reconfigure Software Instance with software type.
  """
196
  time.sleep(1)
197
  if not (updateProxy(config) and requestInstance(config, software_type)):
198 199
    return False

200

Łukasz Nowak's avatar
Łukasz Nowak committed
201
def startProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
202
  """Start Slapproxy server"""
203
  if sup_process.isRunning(config, 'slapproxy'):
204
    return
205
  try:
206
    return sup_process.runProcess(config, "slapproxy")
207 208
  except xmlrpclib.Fault:
    pass
209
  time.sleep(4)
Łukasz Nowak's avatar
Łukasz Nowak committed
210 211 212


def stopProxy(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
213
  """Stop Slapproxy server"""
214
  return sup_process.stopProcess(config, "slapproxy")
Łukasz Nowak's avatar
Łukasz Nowak committed
215 216 217


def removeProxyDb(config):
218
  """Remove Slapproxy database, this is used to initialize proxy for example when
Alain Takoudjou's avatar
Alain Takoudjou committed
219
    configuring new Software Release"""
Łukasz Nowak's avatar
Łukasz Nowak committed
220 221 222
  if os.path.exists(config['database_uri']):
    os.unlink(config['database_uri'])

Marco Mariani's avatar
Marco Mariani committed
223

224
def isSoftwareRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
225
  """
226
    Return True if slapos is still running and false if slapos if not
Alain Takoudjou's avatar
Alain Takoudjou committed
227
  """
228
  return sup_process.isRunning(config, 'slapgrid-sr')
Łukasz Nowak's avatar
Łukasz Nowak committed
229 230


231 232 233 234
def slapgridResultToFile(config, step, returncode, datetime):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  result = {'last_build':datetime, 'success':returncode}
235 236
  with open(file, "w") as f:
    json.dump(result, f)
237 238


239 240 241 242 243 244 245 246 247 248
def getSlapgridResult(config, step):
  filename = step + "_info.json"
  file = os.path.join(config['runner_workdir'], filename)
  if os.path.exists(file):
    result = json.loads(open(file, "r").read())
  else:
    result = {'last_build': 0, 'success':-1}
  return result


249 250 251 252 253
def waitProcess(config, process, step):
  process.wait()
  date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  slapgridResultToFile(config, step, process.returncode, date)

254
def runSlapgridWithLock(config, step, process_name, lock=False):
Alain Takoudjou's avatar
Alain Takoudjou committed
255
  """
256 257 258 259
  * process_name is the name of the process given to supervisord, which will
    run the software or the instance
  * step is one of ('software', 'instance')
  * lock allows to make this function asynchronous or not
Alain Takoudjou's avatar
Alain Takoudjou committed
260
  """
261
  if sup_process.isRunning(config, process_name):
262
    return 1
263

264 265
  log_file = config["%s_log" % step]

266
  # XXX Hackish and unreliable
267 268
  if os.path.exists(log_file):
    os.remove(log_file)
269 270

  if step != "format" and  not updateProxy(config):
271
    return 1
272

273 274
  if step == 'instance' and not requestInstance(config):
    return 1
275

276
  try:
277
    sup_process.runProcess(config, process_name)
278
    if lock:
279
      sup_process.waitForProcessEnd(config, process_name)
280
    #Saves the current compile software for re-use
281 282
    if step == 'software':
      config_SR_folder(config)
283
    return  sup_process.returnCode(config, process_name)
284
  except xmlrpclib.Fault:
285
    return 1
286

Łukasz Nowak's avatar
Łukasz Nowak committed
287

288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
def runSoftwareWithLock(config, lock=False):
  """
    Use Slapgrid to compile current Software Release and wait until
    compilation is done
  """
  return runSlapgridWithLock(config, 'software', 'slapgrid-sr', lock)


def runInstanceWithLock(config, lock=False):
  """
    Use Slapgrid to deploy current Software Release and wait until
    deployment is done.
  """
  return runSlapgridWithLock(config, 'instance', 'slapgrid-cp', lock)


304 305 306 307 308 309 310 311
def runFormatWithLock(config, lock=False):
  """
    Use Slapgrid to deploy current Software Release and wait until
    deployment is done.
  """
  return runSlapgridWithLock(config, 'format', 'slapformat', lock)


312
def config_SR_folder(config):
Marco Mariani's avatar
Marco Mariani committed
313 314
  """Create a symbolik link for each folder in software folder. That allows
    the user to customize software release folder"""
315
  config_name = 'slaprunner.config'
316 317 318 319 320 321 322 323 324 325 326 327 328 329
  def link_to_folder(name, folder):
    destination = os.path.join(config['software_link'], name)
    source = os.path.join(config['software_root'], folder)
    cfg = os.path.join(destination, config_name)
        #create symlink
    if os.path.lexists(destination):
      os.remove(destination)
    os.symlink(source, destination)
        #write config file
    if os.path.exists(source):
      with open(cfg, 'w') as cf:
        cf.write(current_project + '#' + folder)

  # First create the link for current project
330 331
  with open(os.path.join(config['etc_dir'], ".project")) as f:
    current_project = f.read().strip().rstrip('/')
332
  profile = getCurrentSoftwareReleaseProfile(config)
333
  name = getSoftwareReleaseName(config)
334 335 336
  md5sum = md5digest(profile)
  link_to_folder(name, md5sum)
  # check other links
337
  software_link_list = []
338 339 340
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
341 342
      with open(cfg_path) as f:
        cfg = f.read().split("#")
343
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
344
        continue  # there is a broken config file
345
      software_link_list.append(cfg[1])
346 347 348 349
  if os.path.exists(config['software_root']):
    folder_list = os.listdir(config['software_root'])
  else:
    return
350
  if not folder_list:
351 352
    return
  for folder in folder_list:
353
    if folder in software_link_list:
Marco Mariani's avatar
Marco Mariani committed
354
      continue  # this folder is already registered
355
    else:
356
      link_to_folder(folder, folder)
Marco Mariani's avatar
Marco Mariani committed
357

358 359
def loadSoftwareRList(config):
  """Return list (of dict) of Software Release from symbolik SR folder"""
360
  sr_list = []
361 362 363 364
  config_name = 'slaprunner.config'
  for path in os.listdir(config['software_link']):
    cfg_path = os.path.join(config['software_link'], path, config_name)
    if os.path.exists(cfg_path):
365 366
      with open(cfg_path) as f:
        cfg = f.read().split("#")
367
      if len(cfg) != 2:
Marco Mariani's avatar
Marco Mariani committed
368
        continue  # there is a broken config file
369 370
      sr_list.append(dict(md5=cfg[1], path=cfg[0], title=path))
  return sr_list
Łukasz Nowak's avatar
Łukasz Nowak committed
371

Marco Mariani's avatar
Marco Mariani committed
372

373
def isInstanceRunning(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
374
  """
375
    Return True if slapos is still running and False otherwise
Alain Takoudjou's avatar
Alain Takoudjou committed
376
  """
377
  return sup_process.isRunning(config, 'slapgrid-cp')
Łukasz Nowak's avatar
Łukasz Nowak committed
378 379


Alain Takoudjou's avatar
Alain Takoudjou committed
380 381 382 383 384 385 386 387 388 389 390 391
def getProfilePath(projectDir, profile):
  """
  Return the path of the current Software Release `profile`

  Args:
    projectDir: Slaprunner workspace location.
    profile: file to search into the workspace.

  Returns:
    String, path of current Software Release profile
  """
  if not os.path.exists(os.path.join(projectDir, ".project")):
392
    return False
393 394
  with open(os.path.join(projectDir, ".project")) as f:
    projectFolder = f.read()
395
  return os.path.join(projectFolder, profile)
Łukasz Nowak's avatar
Łukasz Nowak committed
396

Marco Mariani's avatar
Marco Mariani committed
397

Łukasz Nowak's avatar
Łukasz Nowak committed
398
def getSlapStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
399
  """Return all Slapos Partitions with associate information"""
Łukasz Nowak's avatar
Łukasz Nowak committed
400 401 402 403 404 405 406
  slap = slapos.slap.slap()
  slap.initializeConnection(config['master_url'])
  partition_list = []
  computer = slap.registerComputer(config['computer_id'])
  try:
    for partition in computer.getComputerPartitionList():
      # Note: Internal use of API, as there is no reflexion interface in SLAP
407
      partition_list.append((partition.getId(), partition._connection_dict.copy()))
Łukasz Nowak's avatar
Łukasz Nowak committed
408 409
  except Exception:
    pass
410
  if partition_list:
411
    for i in range(int(config['partition_amount'])):
412
      slappart_id = '%s%s' % ("slappart", i)
Marco Mariani's avatar
Marco Mariani committed
413
      if not [x[0] for x in partition_list if slappart_id == x[0]]:
414
        partition_list.append((slappart_id, []))
Łukasz Nowak's avatar
Łukasz Nowak committed
415 416
  return partition_list

Marco Mariani's avatar
Marco Mariani committed
417

Łukasz Nowak's avatar
Łukasz Nowak committed
418
def svcStopAll(config):
419
  """Stop all Instance processes on this computer"""
420
  try:
421
    return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
422
                  'stop', 'all']).communicate()[0]
423
  except Exception:
424
    pass
Łukasz Nowak's avatar
Łukasz Nowak committed
425

426 427 428 429 430
def svcStartAll(config):
  """Start all Instance processes on this computer"""
  try:
    return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
                  'start', 'all']).communicate()[0]
431
  except Exception:
432
    pass
Marco Mariani's avatar
Marco Mariani committed
433

434 435
def removeInstanceRootDirectory(config):
  """Clean instance directory"""
436
  if os.path.exists(config['instance_root']):
437 438 439 440 441 442 443 444 445 446 447
    for instance_directory in os.listdir(config['instance_root']):
      instance_directory = os.path.join(config['instance_root'], instance_directory)
      # XXX: hardcoded
      if stat.S_ISSOCK(os.stat(instance_directory).st_mode) or os.path.isfile(instance_directory):
        # Ignore non-instance related files
        continue
      for root, dirs, _ in os.walk(instance_directory):
        for fname in dirs:
          fullPath = os.path.join(root, fname)
          if not os.access(fullPath, os.W_OK):
            # Some directories may be read-only, preventing to remove files in it
448
            os.chmod(fullPath, 0o744)
449
      shutil.rmtree(instance_directory)
450

451 452 453 454 455 456 457 458
def removeCurrentInstance(config):
  if isInstanceRunning(config):
    return "Instantiation in progress, cannot remove instance"

  # Stop all processes
  svcStopAll(config)
  if stopProxy(config):
    removeProxyDb(config)
459
    startProxy(config)
460 461 462 463 464 465 466 467 468 469 470 471 472
  else:
    return "Something went wrong when trying to stop slapproxy."

  # Remove Instance directory and data related to the instance
  try:
    removeInstanceRootDirectory(config)
    param_path = os.path.join(config['etc_dir'], ".parameter.xml")
    if os.path.exists(param_path):
      os.remove(param_path)
  except IOError:
    return "The filesystem couldn't been cleaned properly"
  return True

Marco Mariani's avatar
Marco Mariani committed
473

Łukasz Nowak's avatar
Łukasz Nowak committed
474
def getSvcStatus(config):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
475
  """Return all Softwares Instances process Information"""
476
  result = Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
477
                  'status']).communicate()[0]
478
  regex = "(^unix:.+\.socket)|(^error:)|(^watchdog).*$"
479 480
  supervisord = []
  for item in result.split('\n'):
481
    if item.strip() != "":
Marco Mariani's avatar
Marco Mariani committed
482
      if re.search(regex, item, re.IGNORECASE) is None:
483
        supervisord.append(re.split('[\s,]+', item))
484 485
  return supervisord

Marco Mariani's avatar
Marco Mariani committed
486

487
def getSvcTailProcess(config, process):
488
  """Get log for the specified process
Alain Takoudjou's avatar
Alain Takoudjou committed
489 490 491

  Args:
    config: Slaprunner configuration
492
    process: process name. this value is passed to supervisord.
Alain Takoudjou's avatar
Alain Takoudjou committed
493 494 495
  Returns:
    a string that contains the log of the process.
  """
496
  return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
497 498
                "tail", process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
499

500
def svcStartStopProcess(config, process, action):
Alain Takoudjou's avatar
Alain Takoudjou committed
501 502 503 504 505 506 507
  """Send start or stop process command to supervisord

  Args:
    config: Slaprunner configuration.
    process: process to start or stop.
    action: current state which is used to generate the new process state.
  """
Marco Mariani's avatar
Marco Mariani committed
508 509 510 511 512 513 514
  cmd = {
    'RESTART': 'restart',
    'STOPPED': 'start',
    'RUNNING': 'stop',
    'EXITED': 'start',
    'STOP': 'stop'
  }
515
  return Popen([config['slapos'], 'node', 'supervisorctl', '--cfg', config['configuration_file_path'],
516 517
                cmd[action], process]).communicate()[0]

Marco Mariani's avatar
Marco Mariani committed
518

519 520
def listFolder(config, path):
  """Return the list of folder into path
Alain Takoudjou's avatar
Alain Takoudjou committed
521 522

  Agrs:
523
    path: path of the directory to list
Alain Takoudjou's avatar
Alain Takoudjou committed
524 525 526
  Returns:
    a list that contains each folder name.
  """
527 528 529 530 531 532 533 534
  folderList = []
  folder = realpath(config, path)
  if folder:
    path_list = sorted(os.listdir(folder), key=str.lower)
    for elt in path_list:
      if os.path.isdir(os.path.join(folder, elt)):
        folderList.append(elt)
  return folderList
535

Marco Mariani's avatar
Marco Mariani committed
536

537
def configNewSR(config, projectpath):
Alain Takoudjou's avatar
Alain Takoudjou committed
538 539 540 541 542 543 544 545
  """Configure a Software Release as current Software Release

  Args:
    config: slaprunner configuration
    projectpath: path of the directory that contains the software realease to configure
  Returns:
    True if all is done well, otherwise return false.
  """
546 547
  folder = realpath(config, projectpath)
  if folder:
548 549
    sup_process.stopProcess(config, 'slapgrid-cp')
    sup_process.stopProcess(config, 'slapgrid-sr')
550
    logger.warning("User opened a new SR. Removing all instances...")
551
    removeCurrentInstance(config)
552 553
    with open(os.path.join(config['etc_dir'], ".project"), 'w') as f:
      f.write(projectpath)
554 555 556 557
    return True
  else:
    return False

Marco Mariani's avatar
Marco Mariani committed
558

559
def newSoftware(folder, config, session):
Alain Takoudjou's avatar
Alain Takoudjou committed
560 561 562 563 564 565 566
  """
  Create a new Software Release folder with default profiles

  Args:
    folder: directory of the new software release
    config: slraprunner configuration
    session: Flask session directory"""
567 568
  json = ""
  code = 0
569
  basedir = config['etc_dir']
570
  try:
571 572 573
    folderPath = realpath(config, folder, check_exist=False)
    if folderPath and not os.path.exists(folderPath):
      os.mkdir(folderPath)
574 575
      #load software.cfg and instance.cfg from https://lab.nexedi.com
      software = "https://lab.nexedi.com/nexedi/slapos/raw/master/software/lamp-template/software.cfg"
576 577
      softwareContent = ""
      try:
578
        softwareContent = urlopen(software).read()
579
      except Exception:
580 581
        #Software.cfg and instance.cfg content will be empty
        pass
582 583 584 585 586 587
      with open(os.path.join(folderPath, config['software_profile']), 'w') as f:
        f.write(softwareContent)
      with open(os.path.join(folderPath, config['instance_profile']), 'w') as f:
        pass
      with open(os.path.join(basedir, ".project"), 'w') as f:
        f.write(folder + "/")
588 589 590 591 592
      #Clean sapproxy Database
      stopProxy(config)
      removeProxyDb(config)
      startProxy(config)
      #Stop runngin process and remove existing instance
593
      logger.warning("User created a new SR. Removing all instances...")
594
      removeCurrentInstance(config)
595 596 597
      session['title'] = getProjectTitle(config)
      code = 1
    else:
598
      json = "Bad folder or Directory '%s' already exist, please enter a new name for your software" % folder
Marco Mariani's avatar
Marco Mariani committed
599
  except Exception as e:
600
    json = "Can not create your software, please try again! : %s " % e
601 602
    if os.path.exists(folderPath):
      shutil.rmtree(folderPath)
603 604
  return jsonify(code=code, result=json)

Marco Mariani's avatar
Marco Mariani committed
605

606
def checkSoftwareFolder(path, config):
Alain Takoudjou's avatar
Alain Takoudjou committed
607
  """Check id `path` is a valid Software Release folder"""
608 609
  realdir = realpath(config, path)
  if realdir and os.path.exists(os.path.join(realdir, config['software_profile'])):
610 611 612
    return jsonify(result=path)
  return jsonify(result="")

Marco Mariani's avatar
Marco Mariani committed
613

614
def getProjectTitle(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
615
  """Generate the name of the current software Release (for slaprunner UI)"""
616
  conf = os.path.join(config['etc_dir'], ".project")
617 618 619 620
  # instance_name is optional parameter
  instance_name = config.get('instance_name')
  if instance_name:
    instance_name = '%s - ' % instance_name
621
  if os.path.exists(conf):
622 623
    with open(conf, "r") as f:
      project = f.read().split("/")
Marco Mariani's avatar
Marco Mariani committed
624
    software = project[-2]
625 626
    return '%s%s (%s)' % (instance_name, software, '/'.join(project[:-2]))
  return "%sNo Profile" % instance_name
627

Marco Mariani's avatar
Marco Mariani committed
628

629
def getSoftwareReleaseName(config):
Alain Takoudjou's avatar
Alain Takoudjou committed
630
  """Get the name of the current Software Release"""
631
  sr_profile = os.path.join(config['etc_dir'], ".project")
632
  if os.path.exists(sr_profile):
633
    with open(sr_profile, "r") as f:
634 635 636
      project = f.read().strip().rstrip().split("/")
    # we always use the suffix workspace, so this is the intention
    # behind this method, get the checkout.
Marco Mariani's avatar
Marco Mariani committed
637
    software = project[-2]
638
    return software.replace(' ', '_')
639
  return None
640

641 642 643
def removeSoftwareRootDirectory(config, md5, folder_name):
  """
  Removes all content in the filesystem of the software release specified by md5
Alain Takoudjou's avatar
Alain Takoudjou committed
644 645 646

  Args:
    config: slaprunner configuration
647 648 649
    folder_name: the link name given to the software release
    md5: the md5 filename given by slapgrid to SR folder
  """
650
  path = os.path.join(config['software_root'], md5)
651
  linkpath = os.path.join(config['software_link'], folder_name)
652
  if not os.path.exists(path):
653
    return (0, "Cannot remove software Release: No such file or directory")
654
  if not os.path.exists(linkpath):
655 656
    return (0, "Cannot remove software Release: No such file or directory %s" %
                    ('software_root/' + folder_name))
657
  os.unlink(linkpath)
658
  shutil.rmtree(path)
659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
  return

def removeSoftwareByName(config, md5, folder_name):
  """
  Removes a software release specified by its md5 and its name from the webrunner.
  If the software release is the one of the current running instance, then
  the instance should be stopped.

  Args:
    config: slaprunner configuration
    folder_name: the link name given to the software release
    md5: the md5 filename given by slapgrid to SR folder
  """
  if isSoftwareRunning(config) or isInstanceRunning(config):
    return (0, "Software installation or instantiation in progress, cannot remove")

  if getSoftwareReleaseName(config) == folder_name:
676
    logger.warning("User removed the SR currently used. Removing all instances...")
677 678 679 680 681 682
    removeCurrentInstance(config)

  result = removeSoftwareRootDirectory(config, md5, folder_name)
  if result is not None:
    return result
  return 1, loadSoftwareRList(config)
683

Marco Mariani's avatar
Marco Mariani committed
684

685 686 687 688 689 690 691 692 693 694 695 696
def tail(f, lines=20):
  """
  Returns the last `lines` lines of file `f`. It is an implementation of tail -f n.
  """
  BUFSIZ = 1024
  f.seek(0, 2)
  bytes = f.tell()
  size = lines + 1
  block = -1
  data = []
  while size > 0 and bytes > 0:
      if bytes - BUFSIZ > 0:
697 698 699 700
          # Seek back one whole BUFSIZ
          f.seek(block * BUFSIZ, 2)
          # read BUFFER
          data.insert(0, f.read(BUFSIZ))
701
      else:
702
          # file too small, start from begining
Marco Mariani's avatar
Marco Mariani committed
703
          f.seek(0, 0)
704 705
          # only read what was not read
          data.insert(0, f.read(bytes))
706 707 708 709
      linesFound = data[0].count('\n')
      size -= linesFound
      bytes -= BUFSIZ
      block -= 1
Marco Mariani's avatar
Marco Mariani committed
710
  return '\n'.join(''.join(data).splitlines()[-lines:])
711

Marco Mariani's avatar
Marco Mariani committed
712

713
def readFileFrom(f, lastPosition, limit=20000):
714 715 716
  """
  Returns the last lines of file `f`, from position lastPosition.
  and the last position
717
  limit = max number of characters to read
718 719 720
  """
  BUFSIZ = 1024
  f.seek(0, 2)
Marco Mariani's avatar
Marco Mariani committed
721
  # XXX-Marco do now shadow 'bytes'
722 723
  bytes = f.tell()
  block = -1
724
  data = b""
725
  length = bytes
Marco Mariani's avatar
Marco Mariani committed
726 727
  truncated = False  # True if a part of log data has been truncated
  if (lastPosition <= 0 and length > limit) or (length - lastPosition > limit):
728
    lastPosition = length - limit
729
    truncated = True
730 731
  size = bytes - lastPosition
  while bytes > lastPosition:
Marco Mariani's avatar
Marco Mariani committed
732
    if abs(block * BUFSIZ) <= size:
733 734 735 736
      # Seek back one whole BUFSIZ
      f.seek(block * BUFSIZ, 2)
      data = f.read(BUFSIZ) + data
    else:
Marco Mariani's avatar
Marco Mariani committed
737
      margin = abs(block * BUFSIZ) - size
738
      if length < BUFSIZ:
Marco Mariani's avatar
Marco Mariani committed
739
        f.seek(0, 0)
740
      else:
741 742
        seek = block * BUFSIZ + margin
        f.seek(seek, 2)
743 744 745
      data = f.read(BUFSIZ - margin) + data
    bytes -= BUFSIZ
    block -= 1
Marco Mariani's avatar
Marco Mariani committed
746 747 748 749 750 751
  return {
    'content': data,
    'position': length,
    'truncated': truncated
  }

752

753 754
text_range = str2bytes(''.join(map(chr, [7, 8, 9, 10, 12, 13, 27]
                                        + list(range(0x20, 0x100)))))
755 756 757
def isText(file):
  """Return True if the mimetype of file is Text"""
  try:
758 759 760
    with open(file, 'rb') as f:
      return not f.read(1024).translate(None, text_range)
  except Exception:
761
    return False
762

Marco Mariani's avatar
Marco Mariani committed
763

764 765
def md5sum(file):
  """Compute md5sum of `file` and return hexdigest value"""
766
  # XXX-Marco: returning object or False boolean is an anti-pattern. better to return object or None
767 768 769
  if os.path.isdir(file):
    return False
  try:
770
    m = hashlib.md5()
771 772 773 774 775 776
    with open(file, 'rb') as fh:
      while True:
        data = fh.read(8192)
        if not data:
          break
        m.update(data)
777
    return m.hexdigest()
778
  except Exception:
779 780
    return False

Marco Mariani's avatar
Marco Mariani committed
781

782
def realpath(config, path, check_exist=True):
783 784 785 786
  """
  Get realpath of path or return False if user is not allowed to access to
  this file.
  """
787 788
  split_path = path.split('/')
  key = split_path[0]
789 790 791
  virtual_path_list = ('software_root', 'instance_root', 'workspace',
    'runner_workdir', 'software_link')
  if key not in virtual_path_list:
792
    return ''
793
  allow_list = {path: config[path] for path in virtual_path_list if path in config}
Marco Mariani's avatar
Marco Mariani committed
794 795 796 797
  del split_path[0]
  path = os.path.join(allow_list[key], *split_path)
  if check_exist:
    if os.path.exists(path):
798
      return path
Marco Mariani's avatar
Marco Mariani committed
799
    else:
800
      return ''
Marco Mariani's avatar
Marco Mariani committed
801 802 803
  else:
    return path

804 805

def readParameters(path):
Alain Takoudjou's avatar
Alain Takoudjou committed
806 807 808 809 810 811
  """Read Instance parameters stored into a local file.

  Agrs:
    path: path of the xml file that contains parameters

  Return:
Marco Mariani's avatar
Marco Mariani committed
812
    a dictionary of instance parameters."""
813 814
  if os.path.exists(path):
    try:
815
      xmldoc = minidom.parse(path)
816
      obj = {}
817
      for elt in xmldoc.childNodes:
Marco Mariani's avatar
Marco Mariani committed
818
        sub_obj = {}
819 820
        for subnode in elt.childNodes:
          if subnode.nodeType != subnode.TEXT_NODE:
Marco Mariani's avatar
Marco Mariani committed
821
            sub_obj[str(subnode.getAttribute('id'))] = subnode.childNodes[0].data  # .decode('utf-8').decode('utf-8')
Marco Mariani's avatar
Marco Mariani committed
822 823
            obj[str(elt.tagName)] = sub_obj
      return obj
824
    except Exception as e:
825 826
      return str(e)
  else:
827
    return "No such file or directory: %s" % path
828

829 830 831 832
def isSoftwareReleaseCompleted(config):
  software_name = getSoftwareReleaseName(config)
  if software_name is None:
    return False
833 834
  elif os.path.exists(os.path.join(config['software_link'],
      software_name, '.completed')):
835 836 837
    return True
  else:
    return False
838

839 840 841 842
def isSoftwareReleaseReady(config):
  """Return 1 if the Software Release has
  correctly been deployed, 0 if not,
  and 2 if it is currently deploying"""
843 844
  auto_deploy = config['auto_deploy'] in TRUE_VALUES
  auto_run = config['autorun'] in TRUE_VALUES
845
  project = os.path.join(config['etc_dir'], '.project')
846
  if not ( os.path.exists(project) and (auto_run or auto_deploy) ):
847
    return "0"
848
  updateInstanceParameter(config)
849
  if isSoftwareReleaseCompleted(config):
850
    if auto_run:
851
      runSlapgridUntilSuccess(config, 'instance')
852 853 854 855
    return "1"
  else:
    if isSoftwareRunning(config):
      return "2"
856
    elif auto_deploy:
857
      runSoftwareWithLock(config)
858
      if auto_run:
859
        runSlapgridUntilSuccess(config, 'instance')
860 861 862
      return "2"
    else:
      return "0"
863

864

865
def cloneDefaultGit(config):
866
  """Test if the default git has been downloaded yet
867
  If not, download it in read-only mode"""
868
  default_git = os.path.join(config['runner_workdir'],
869
    'project', 'default_repo')
870 871 872
  if not os.path.exists(default_git):
    data = {'path': default_git,
            'repo': config['default_repo'],
873 874
    }
    cloneRepo(data)
875

876

877 878 879 880
def buildAndRun(config):
  runSoftwareWithLock(config)
  runInstanceWithLock(config)

881

882
def runSlapgridUntilSuccess(config, step, bang=False):
883
  """Run slapos several times,
884
  in the maximum of the constant MAX_RUN_~~~~"""
885
  params = getBuildAndRunParams(config)
886
  if step == "instance":
887 888 889 890 891 892 893 894 895
    if bang:
      # We'd prefer that 'node instance' is invoked with --all but that's
      # not possible with current design. The alternative is to bang,
      # assuming that the requested partition is partition 0.
      slap = slapos.slap.slap()
      slap.initializeConnection(config['master_url'])
      partition = slap.registerComputerPartition(
        config['computer_id'], 'slappart0')
      partition.bang('manual instantiation')
896
    max_tries = (params['max_run_instance'] if params['run_instance'] else 0)
897 898
    runSlapgridWithLock = runInstanceWithLock
  elif step == "software":
899
    max_tries = (params['max_run_software'] if params['run_software'] else 0)
900 901
    runSlapgridWithLock = runSoftwareWithLock
  else:
902
    return -1
903
  counter_file = os.path.join(config['runner_workdir'], '.turn-left')
904 905
  with open(counter_file, 'w+') as f:
    f.write(str(max_tries))
906
  counter = max_tries
907
  slapgrid = True
908
  # XXX-Nico runSoftwareWithLock can return 0 or False (0==False)
909 910
  while counter > 0:
    counter -= 1
911 912 913
    slapgrid = runSlapgridWithLock(config, lock=True)
    # slapgrid == 0 because EXIT_SUCCESS == 0
    if slapgrid == 0:
914
      break
915 916
    with open(counter_file) as f:
      times_left = int(f.read()) - 1
917
    if times_left > 0 :
918 919
      with open(counter_file, 'w+') as f:
        f.write(str(times_left))
920 921 922
      counter = times_left
    else :
      counter = 0
923
  max_tries -= counter
924 925
  # run instance only if we are deploying the software release,
  # if it is defined so, and sr is correctly deployed
926
  if step == "software" and params['run_instance'] and slapgrid == 0:
927
    return (max_tries, runSlapgridUntilSuccess(config, "instance", bang))
928 929
  else:
    return max_tries
930 931


932 933
def setupDefaultSR(config):
  """If a default_sr is in the parameters,
934 935
  and no SR is deployed yet, setup it
  also run SR and Instance if required"""
936 937 938
  project = os.path.join(config['etc_dir'], '.project')
  if not os.path.exists(project) and config['default_sr'] != '':
    configNewSR(config, config['default_sr'])
939
  if config['auto_deploy']:
940
    _thread.start_new_thread(buildAndRun, (config,))
941 942 943 944 945 946 947


def setMiniShellHistory(config, command):
  history_max_size = 10
  command = command + "\n"
  history_file = config['minishell_history_file']
  if os.path.exists(history_file):
948 949
    with open(history_file, 'r') as f:
      history = f.readlines()
950 951 952 953 954
    if len(history) >= history_max_size:
      del history[0]
  else:
    history = []
  history.append(command)
955 956
  with open(history_file, 'w') as f:
    f.write(''.join(history))