Commit 7c87afa6 authored by Alain Takoudjou's avatar Alain Takoudjou

Merge branch 'master' into 1.0

parents ea1bce1f e1a9a751
Changes Changes
======= =======
1.0.44 (2016-12-30)
-------------------
* pbs: handles the fact that some parameters are not present when slaves are down
* recipe: allow usage of pidfile in wrapper recipe
* sshd: fix generation of authorized_keys
1.0.43 (2016-11-24) 1.0.43 (2016-11-24)
------------------- -------------------
......
...@@ -11,9 +11,9 @@ parts = ...@@ -11,9 +11,9 @@ parts =
[ca-certificates] [ca-certificates]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20160104.tar.xz url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130.tar.xz
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
md5sum = d9665a83d0d3ef8176a38e6aa20458e9 md5sum = 1a0a3a1b3390dc83affed4b0c2ae1c05
patches = patches =
${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2 ${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
......
...@@ -6,8 +6,8 @@ parts = garbage-collector ...@@ -6,8 +6,8 @@ parts = garbage-collector
[garbage-collector] [garbage-collector]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
md5sum = 5e230029f802d0ac633b1d9b3d4934c9 md5sum = 6f77f9fff5fb5bf96adfc1e93cd035b6
url = http://www.hboehm.info/gc/gc_source/gc-7.2e.tar.gz url = http://www.hboehm.info/gc/gc_source/gc-7.2g.tar.gz
configure-options = configure-options =
--enable-cplusplus --enable-cplusplus
--disable-gcj-support --disable-gcj-support
......
...@@ -17,8 +17,8 @@ parts = ...@@ -17,8 +17,8 @@ parts =
[git] [git]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://www.kernel.org/pub/software/scm/git/git-2.9.2.tar.xz url = https://www.kernel.org/pub/software/scm/git/git-2.11.0.tar.xz
md5sum = dc5eb7ff2ca4641c6038f51ae49cad0f md5sum = dd4e3360e28aec5bb902fb34dd7fce3b
configure-options = configure-options =
--with-curl=${curl:location} --with-curl=${curl:location}
--with-openssl=${openssl:location} --with-openssl=${openssl:location}
......
...@@ -14,8 +14,8 @@ extends = ...@@ -14,8 +14,8 @@ extends =
[groonga] [groonga]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-6.0.9.tar.gz url = http://packages.groonga.org/source/groonga/groonga-6.1.1.tar.gz
md5sum = 792d6a70536c888b2007f6ef55e9e1c0 md5sum = f2dedb4b1a536a5e11a2b9a35664125b
# temporary patch to respect more tokens in natural language mode. # temporary patch to respect more tokens in natural language mode.
patches = patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
...@@ -5,8 +5,8 @@ parts = icu4c ...@@ -5,8 +5,8 @@ parts = icu4c
[icu4c] [icu4c]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = http://download.icu-project.org/files/icu4c/4.8.1.1/icu4c-4_8_1_1-src.tgz url = http://download.icu-project.org/files/icu4c/58.2/icu4c-58_2-src.tgz
md5sum = ea93970a0275be6b42f56953cd332c17 md5sum = fac212b32b7ec7ab007a12dff1f3aea1
configure-command = source/configure configure-command = source/configure
configure-options = configure-options =
--prefix=${:location} --prefix=${:location}
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
extends = extends =
../libjpeg/buildout.cfg ../libjpeg/buildout.cfg
../jbigkit/buildout.cfg ../jbigkit/buildout.cfg
../patch/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
parts = parts =
...@@ -10,12 +9,8 @@ parts = ...@@ -10,12 +9,8 @@ parts =
[libtiff] [libtiff]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 4.0.3 url = http://download.osgeo.org/libtiff/tiff-4.0.7.tar.gz
url = http://www.imagemagick.org/download/delegates/tiff-${:version}.tar.gz md5sum = 77ae928d2c6b7fb46a21c3a29325157b
md5sum = 051c1068e6a0627f461948c365290410
patch-options = -p1
patches =
${:_profile_base_location_}/tiff_4.0.3-12.3.debian.patch#c246dc095f57f869b35cf40a32aa8366
configure-options = configure-options =
--disable-static --disable-static
--without-x --without-x
......
This diff is collapsed.
...@@ -22,9 +22,9 @@ parts = ...@@ -22,9 +22,9 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 10.1.19 version = 10.1.20
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve
md5sum = e22e491eb3766bc3151cc0e96a7531c2 md5sum = 623226918fd711e414ad240287284b5b
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
patch-options = -p0 patch-options = -p0
patches = patches =
...@@ -70,8 +70,8 @@ post-install = ...@@ -70,8 +70,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users. # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/ # http://mroonga.github.com/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-6.09.tar.gz url = http://packages.groonga.org/source/mroonga/mroonga-6.11.tar.gz
md5sum = ac8672d81d6a8ceb13c2a9092f19541f md5sum = 1c30bb6d89dbee0cc2f9b0a5cfaaccf0
pre-configure = pre-configure =
mkdir fake_mariadb_source && mkdir fake_mariadb_source &&
ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
......
...@@ -15,8 +15,8 @@ parts = ...@@ -15,8 +15,8 @@ parts =
[openssh] [openssh]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
md5sum = dfadd9f035d38ce5d58a3bf130b86d08 md5sum = b2db2a83caf66a208bb78d6d287cdaa3
url = http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.3p1.tar.gz url = http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.4p1.tar.gz
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
#patch-options = -p1 #patch-options = -p1
patches = patches =
......
...@@ -10,8 +10,8 @@ parts = ...@@ -10,8 +10,8 @@ parts =
[openvpn] [openvpn]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://swupdate.openvpn.org/community/releases/openvpn-2.3.12.tar.xz url = https://swupdate.openvpn.org/community/releases/openvpn-2.3.14.tar.xz
md5sum = 63326bab2ebb9efe3c7becaa4f15e1c1 md5sum = 70fc1e9c74ba2e6d7f7e320313dc3a7b
configure-options = configure-options =
--disable-dependency-tracking --disable-dependency-tracking
--disable-plugin-auth-pam --disable-plugin-auth-pam
...@@ -19,9 +19,9 @@ environment = ...@@ -19,9 +19,9 @@ environment =
PATH=${xz-utils:location}/bin:%(PATH)s PATH=${xz-utils:location}/bin:%(PATH)s
LZO_LIBS=-L${lzo:location}/lib -llzo2 LZO_LIBS=-L${lzo:location}/lib -llzo2
LZO_CFLAGS=-I${lzo:location}/include LZO_CFLAGS=-I${lzo:location}/include
OPENSSL_SSL_LIBS="-L${openssl:location}/lib -lssl" OPENSSL_SSL_LIBS=-L${openssl:location}/lib -lssl
OPENSSL_SSL_CFLAGS="-I${openssl:location}/include/" OPENSSL_SSL_CFLAGS=-I${openssl:location}/include
OPENSSL_CRYPTO_LIBS="-L${openssl:location}/lib -lcrypto" OPENSSL_CRYPTO_LIBS=-L${openssl:location}/lib -lcrypto
OPENSSL_CRYPTO_CFLAGS="-I${openssl:location}/include" OPENSSL_CRYPTO_CFLAGS=-I${openssl:location}/include
LDFLAGS =-Wl,-rpath=${lzo:location}/lib -Wl,-rpath=${flex:location}/lib -Wl,-rpath=${openssl:location}/lib LDFLAGS =-Wl,-rpath=${lzo:location}/lib -Wl,-rpath=${flex:location}/lib -Wl,-rpath=${openssl:location}/lib
CPPFLAGS=-I${lzo:location}/include -I${flex:location}/include CPPFLAGS=-I${lzo:location}/include -I${flex:location}/include
...@@ -28,9 +28,9 @@ python = python2.7 ...@@ -28,9 +28,9 @@ python = python2.7
[python2.7] [python2.7]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
package_version = 2.7.12 package_version = 2.7.13
package_version_suffix = package_version_suffix =
md5sum = 57dffcee9cee8bb2ab5f82af1d8e9a69 md5sum = 53b43534153bb2a0363f08bae8b9d990
# This is actually the default setting for prefix, but we can't use it in # This is actually the default setting for prefix, but we can't use it in
# other settings in this part if we don't set it explicitly here. # other settings in this part if we don't set it explicitly here.
......
...@@ -19,8 +19,8 @@ extends = ...@@ -19,8 +19,8 @@ extends =
[kvm] [kvm]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3. # qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-2.7.0.tar.bz2 url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2
md5sum = 08d4d06d1cb598efecd796137f4844ab md5sum = 17940dce063b6ce450a12e719a6c9c43
configure-options = configure-options =
--target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu" --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
--enable-system --enable-system
......
...@@ -10,7 +10,7 @@ parts = rina-tools ...@@ -10,7 +10,7 @@ parts = rina-tools
[irati-stack] [irati-stack]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://github.com/jmuchemb/irati-stack.git repository = https://github.com/jmuchemb/irati-stack.git
revision = af3cd8350ec43ab2d309c785daf0dd16d3835005 revision = 2e1cdbcb497f9b088fa5a9ff9489df2f59d81d07
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[rina-tools] [rina-tools]
......
...@@ -2,21 +2,23 @@ ...@@ -2,21 +2,23 @@
extends = extends =
../fontconfig/buildout.cfg ../fontconfig/buildout.cfg
../tar/buildout.cfg
../xorg/buildout.cfg ../xorg/buildout.cfg
parts = parts =
tar
wkhtmltopdf wkhtmltopdf
[wkhtmltopdf] [wkhtmltopdf]
recipe = slapos.recipe.build recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64). # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = http://wkhtmltopdf.googlecode.com/files/wkhtmltopdf-0.11.0_rc1-static-amd64.tar.bz2 url_x86-64 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz
url_x86 = http://wkhtmltopdf.googlecode.com/files/wkhtmltopdf-0.11.0_rc1-static-i386.tar.bz2 url_x86 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic-i386.tar.xz
# supported architectures md5sums # supported architectures md5sums
md5sum_x86 = d796cfb66e45673d8f6728b61f34d5e0 md5sum_x86 = ce1a2c0b2cf786ccc5d5828c42c99ddd
md5sum_x86-64 = 02a2c6963728b69b8e329dcbf1f4c7e1 md5sum_x86-64 = 96b7306cebb9e65355f69f7ab63df68b
# script to install. # script to install.
script = script =
...@@ -28,14 +30,19 @@ script = ...@@ -28,14 +30,19 @@ script =
platform = guessPlatform() platform = guessPlatform()
url = self.options['url_' + platform] url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform] md5sum = self.options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum)) path = self.download(url, md5sum)
shutil.move(extract_dir, location) import tempfile
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory ' + repr(extract_dir))
call(["${tar:location}/bin/tar", "xJf", path, "-C", extract_dir])
shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
wrapper_location = os.path.join("%(location)s", "wkhtmltopdf") wrapper_location = os.path.join("%(location)s", "wkhtmltopdf")
wrapper = open(wrapper_location, 'w') wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash wrapper.write("""#!${dash:location}/bin/dash
cd %(location)s export LD_LIBRARY_PATH=%(location)s/lib/:${libXrender:location}/lib/:${fontconfig:location}/lib/:${libX11:location}/lib/:${libXext:location}/lib/
export LD_LIBRARY_PATH=%(location)s:${libXrender:location}/lib/:${fontconfig:location}/lib/:${libX11:location}/lib/:${libXext:location}/lib/
export PATH=${fontconfig:location}/bin:$PATH export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/wkhtmltopdf-""" + WK_SUFIX_MAP[platform]+ """ $*""") exec %(location)s/bin/wkhtmltopdf "$@"
""")
wrapper.close() wrapper.close()
os.chmod(wrapper_location, 0755) os.chmod(wrapper_location, 0755)
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.43' version = '1.0.44'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \ long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n" open("CHANGES.rst").read() + "\n"
......
...@@ -24,8 +24,7 @@ ...@@ -24,8 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import os import errno, os
import itertools
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
class KnownHostsFile(dict): class KnownHostsFile(dict):
...@@ -55,40 +54,6 @@ class KnownHostsFile(dict): ...@@ -55,40 +54,6 @@ class KnownHostsFile(dict):
def __exit__(self, exc_type, exc_value, traceback): def __exit__(self, exc_type, exc_value, traceback):
self._dump() self._dump()
class AuthorizedKeysFile(object):
def __init__(self, filename):
self.filename = filename
def append(self, key):
"""Append the key to the file if the key's not in the file
"""
# Create the file it it does not exist
try:
file_ = os.open(self.filename, os.O_CREAT | os.O_EXCL)
os.close(file_)
except:
pass
with open(self.filename, 'r') as keyfile:
# itertools.imap avoid loading all the authorized_keys file in
# memory which would be counterproductive.
present = (key.strip() in itertools.imap(lambda k: k.strip(),
keyfile))
try:
keyfile.seek(-1, os.SEEK_END)
ended_by_newline = (keyfile.read() == '\n')
except IOError:
ended_by_newline = True
if not present:
with open(self.filename, 'a') as keyfile:
if not ended_by_newline:
keyfile.write('\n')
keyfile.write(key.strip())
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
...@@ -164,37 +129,29 @@ class Client(GenericBaseRecipe): ...@@ -164,37 +129,29 @@ class Client(GenericBaseRecipe):
return [wrapper] return [wrapper]
def keysplit(s):
"""
Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
and return the individual key_type + key strings.
TODO: handle comments in ssh keys, which are generated
by default at key creation.
"""
s = s.replace('\n', ' ')
si = iter(s.split(' '))
while True:
key_type = next(si)
if key_type == '':
continue
try:
key_value = next(si)
except StopIteration:
# odd number of elements, should not happen
break
yield '%s %s' % (key_type, key_value)
class AddAuthorizedKey(GenericBaseRecipe): class AddAuthorizedKey(GenericBaseRecipe):
def install(self): def install(self):
key = self.options['key']
ssh = self.createDirectory(self.options['home'], '.ssh') ssh = self.createDirectory(self.options['home'], '.ssh')
filename = os.path.join(ssh, 'authorized_keys')
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys')) try:
for key in keysplit(self.options['key']): with open(filename) as f:
# XXX key might actually be the string 'None' or 'null' if f.read() == key:
authorized_keys.append(key) return [filename]
except IOError as e:
return [authorized_keys.filename] if e.errno != errno.ENOENT:
raise
# Atomic update.
tmp = filename + '.new'
try:
with open(tmp, 'w') as f:
f.write(key)
os.rename(tmp, filename)
finally:
try:
os.remove(tmp)
except OSError as e:
if e.errno != errno.ENOENT:
raise
return [filename]
...@@ -43,7 +43,6 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -43,7 +43,6 @@ class NeoBaseRecipe(GenericBaseRecipe):
# useful, as per NEO deploying constraints. # useful, as per NEO deploying constraints.
raise UserError('"masters" parameter is mandatory') raise UserError('"masters" parameter is mandatory')
option_list = [ option_list = [
options['binary'],
'-l', options['logfile'], '-l', options['logfile'],
'-m', options['masters'], '-m', options['masters'],
'-b', self._getBindingAddress(), '-b', self._getBindingAddress(),
...@@ -59,9 +58,9 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -59,9 +58,9 @@ class NeoBaseRecipe(GenericBaseRecipe):
'--key', etc + 'neo.key', '--key', etc + 'neo.key',
) )
option_list.extend(self._getOptionList()) option_list.extend(self._getOptionList())
return [self.createPythonScript( return [self.createWrapper(
options['wrapper'], options['wrapper'],
'slapos.recipe.librecipe.execute.execute', options['binary'],
option_list option_list
)] )]
......
...@@ -65,6 +65,7 @@ def promise(args): ...@@ -65,6 +65,7 @@ def promise(args):
class Recipe(GenericSlapRecipe, Notify, Callback): class Recipe(GenericSlapRecipe, Notify, Callback):
def _options(self, options): def _options(self, options):
options['rdiff-backup-data-folder'] = ""
if 'slave-instance-list' in options: if 'slave-instance-list' in options:
for slave in json.loads(options['slave-instance-list']): for slave in json.loads(options['slave-instance-list']):
if slave['type'] == 'pull': if slave['type'] == 'pull':
...@@ -87,9 +88,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -87,9 +88,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
--restore-as-of now \\ --restore-as-of now \\
--force \\ --force \\
%(local_dir)s \\ %(local_dir)s \\
%(remote_dir)s; do %(remote_dir)s
echo "repeating rdiff-backup..."
sleep 10
""") """)
template_dict = { template_dict = {
......
...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe): ...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
wait_files = self.options.get('wait-for-files') wait_files = self.options.get('wait-for-files')
environment = self.options.get('environment') environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra') parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile')
if not wait_files and not environment: if not wait_files and not environment:
# Create a simple wrapper as shell script # Create a simple wrapper as shell script
...@@ -45,6 +46,7 @@ class Recipe(GenericBaseRecipe): ...@@ -45,6 +46,7 @@ class Recipe(GenericBaseRecipe):
command=command_line[0], command=command_line[0],
parameters=command_line[1:], parameters=command_line[1:],
parameters_extra=parameters_extra, parameters_extra=parameters_extra,
pidfile=pidfile,
)] )]
# More complex needs: create a Python script as wrapper # More complex needs: create a Python script as wrapper
...@@ -72,5 +74,7 @@ class Recipe(GenericBaseRecipe): ...@@ -72,5 +74,7 @@ class Recipe(GenericBaseRecipe):
name=wrapper_path, name=wrapper_path,
command=python_script, command=python_script,
parameters=[], parameters=[],
parameters_extra=parameters_extra) ] parameters_extra=parameters_extra,
pidfile=pidfile,
)]
import unittest
from slapos.recipe.dropbear import keysplit
class TestAuthorizedKey(unittest.TestCase):
def test_keysplit_on_2_keys(self):
given_key_list = [
"""ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDX0juuOAibEjr3K116XZy/kc+6TrxsduaiAKSKgq+ktlgL2hbLmDKaXXbF6xbp8qUNUH3rkMXY3ujFRVGP+MMNkasaxlRBS/rXVwj4uQvSkOQXLWCOkHlwIsrS4xZ3yBXzYDz3yOXL4wQn2wzXrbR3ByxtAoZ/puLGJNvYC3i+LH33cif/SUkEWCyTT5fbLDb5rswjWgrf3v+MzXxuRYZl0rjvi31Ku/mIpLl7Jb7K8a71iQD7xUbEd6GxUORzWNbTrn4rQbUhybOOFc8PnMQJ6wb5vYvwr3UraJq64rH9WRhHak8a25mmBN00h4izEVC+AyrrdZ7Txfa0F2vGxyj/MM56NMAQW4dTkKLju71AXWZkhg2I5kiBgjGmGWyXDgxoKjJVHMJziKf9tqg1pFeDlHI0Q38zmYgCaORNpC+1nG2ydx1/gB9kTky+risJmrc81YlWtZ7lEjpCtSmcAEFIGDHvy0MbZu8UwvykKpGWASIgi4gTapPXjgcvb47cO0JOSZlCwzqrmB3qv/PChOsrpdUljRoB0r1ciWAlHWq2vMEWGQD3uJu6KQ3Dxl7eQp4IFlO65uqtyxkz/t72JeoEAn7lHFd19gGn2QikaD0MPNVL1y/py49WDCPG5tvPubJE6IDky0IrFCFuKegSUtQ0N9BAUAAmesTddSzIPVbr5Q==
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDm2q2ROZIatffz8f1d/9poPlFwflQwDv3EM8Ey674I9GFO9tHQTl2Y8Cp8KlJEGIwiSHJH4J+0F/MrqmwbShHA/R+ocoTAv3nubWGqdKll4o7/wMfHh/7l9H1nqvY3McXO+yjbjbpA2fGFEzQX+Z+j6g+mzrlcYnPB0Bjpc/sPGeOSCfTGmw+BjsxkkwQEvYarZriM1eCz9i7aYJ87OSFgDyo96Dj9ke7EHOESYV0LzxN2WLN/3pYGTdr3Dg18krupCP+X3FAdgd+WP9TXKEcMFHcVs6y7Lyzis52irgEpSysqoOImQ8hdyLs/0oamXCnePoAzzHeTlw8IpUvhpTSYYtWMtCv9velIvxHsnSu87k3HD1jsZ/SZtsrrUwEFMs/9Z8wZQ6q5/GAKo6LRsXxv+BXpXNwc6K3qqxMi5D4TD0p7GBOuz6xmpzScr9eQVn1OqoekDvkdzeXmkOZC01XrJJfD1GA8fjUk7M2OTWeINNSNFiL3ovgz6cMDSuJq39vMPeH2CBW9ozjtMmODlmwEot4FHu3mq5eoajrJnpmpi7oJ6ks8icafs3GTIJDg6sAF3M3oxMXfVVKEUaqvYihq0u9T1qBjU4/raTWjUj2/DhVDNa0gX6OE/xUIdMZMqeeVHndFqb1lkORiPoXcTrD734HNvrmW73rI5TBWMK86RQ==""",
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDX0juuOAibEjr3K116XZy/kc+6TrxsduaiAKSKgq+ktlgL2hbLmDKaXXbF6xbp8qUNUH3rkMXY3ujFRVGP+MMNkasaxlRBS/rXVwj4uQvSkOQXLWCOkHlwIsrS4xZ3yBXzYDz3yOXL4wQn2wzXrbR3ByxtAoZ/puLGJNvYC3i+LH33cif/SUkEWCyTT5fbLDb5rswjWgrf3v+MzXxuRYZl0rjvi31Ku/mIpLl7Jb7K8a71iQD7xUbEd6GxUORzWNbTrn4rQbUhybOOFc8PnMQJ6wb5vYvwr3UraJq64rH9WRhHak8a25mmBN00h4izEVC+AyrrdZ7Txfa0F2vGxyj/MM56NMAQW4dTkKLju71AXWZkhg2I5kiBgjGmGWyXDgxoKjJVHMJziKf9tqg1pFeDlHI0Q38zmYgCaORNpC+1nG2ydx1/gB9kTky+risJmrc81YlWtZ7lEjpCtSmcAEFIGDHvy0MbZu8UwvykKpGWASIgi4gTapPXjgcvb47cO0JOSZlCwzqrmB3qv/PChOsrpdUljRoB0r1ciWAlHWq2vMEWGQD3uJu6KQ3Dxl7eQp4IFlO65uqtyxkz/t72JeoEAn7lHFd19gGn2QikaD0MPNVL1y/py49WDCPG5tvPubJE6IDky0IrFCFuKegSUtQ0N9BAUAAmesTddSzIPVbr5Q== ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDm2q2ROZIatffz8f1d/9poPlFwflQwDv3EM8Ey674I9GFO9tHQTl2Y8Cp8KlJEGIwiSHJH4J+0F/MrqmwbShHA/R+ocoTAv3nubWGqdKll4o7/wMfHh/7l9H1nqvY3McXO+yjbjbpA2fGFEzQX+Z+j6g+mzrlcYnPB0Bjpc/sPGeOSCfTGmw+BjsxkkwQEvYarZriM1eCz9i7aYJ87OSFgDyo96Dj9ke7EHOESYV0LzxN2WLN/3pYGTdr3Dg18krupCP+X3FAdgd+WP9TXKEcMFHcVs6y7Lyzis52irgEpSysqoOImQ8hdyLs/0oamXCnePoAzzHeTlw8IpUvhpTSYYtWMtCv9velIvxHsnSu87k3HD1jsZ/SZtsrrUwEFMs/9Z8wZQ6q5/GAKo6LRsXxv+BXpXNwc6K3qqxMi5D4TD0p7GBOuz6xmpzScr9eQVn1OqoekDvkdzeXmkOZC01XrJJfD1GA8fjUk7M2OTWeINNSNFiL3ovgz6cMDSuJq39vMPeH2CBW9ozjtMmODlmwEot4FHu3mq5eoajrJnpmpi7oJ6ks8icafs3GTIJDg6sAF3M3oxMXfVVKEUaqvYihq0u9T1qBjU4/raTWjUj2/DhVDNa0gX6OE/xUIdMZMqeeVHndFqb1lkORiPoXcTrD734HNvrmW73rI5TBWMK86RQ==",
]
expected_result = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDX0juuOAibEjr3K116XZy/kc+6TrxsduaiAKSKgq+ktlgL2hbLmDKaXXbF6xbp8qUNUH3rkMXY3ujFRVGP+MMNkasaxlRBS/rXVwj4uQvSkOQXLWCOkHlwIsrS4xZ3yBXzYDz3yOXL4wQn2wzXrbR3ByxtAoZ/puLGJNvYC3i+LH33cif/SUkEWCyTT5fbLDb5rswjWgrf3v+MzXxuRYZl0rjvi31Ku/mIpLl7Jb7K8a71iQD7xUbEd6GxUORzWNbTrn4rQbUhybOOFc8PnMQJ6wb5vYvwr3UraJq64rH9WRhHak8a25mmBN00h4izEVC+AyrrdZ7Txfa0F2vGxyj/MM56NMAQW4dTkKLju71AXWZkhg2I5kiBgjGmGWyXDgxoKjJVHMJziKf9tqg1pFeDlHI0Q38zmYgCaORNpC+1nG2ydx1/gB9kTky+risJmrc81YlWtZ7lEjpCtSmcAEFIGDHvy0MbZu8UwvykKpGWASIgi4gTapPXjgcvb47cO0JOSZlCwzqrmB3qv/PChOsrpdUljRoB0r1ciWAlHWq2vMEWGQD3uJu6KQ3Dxl7eQp4IFlO65uqtyxkz/t72JeoEAn7lHFd19gGn2QikaD0MPNVL1y/py49WDCPG5tvPubJE6IDky0IrFCFuKegSUtQ0N9BAUAAmesTddSzIPVbr5Q==", "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDm2q2ROZIatffz8f1d/9poPlFwflQwDv3EM8Ey674I9GFO9tHQTl2Y8Cp8KlJEGIwiSHJH4J+0F/MrqmwbShHA/R+ocoTAv3nubWGqdKll4o7/wMfHh/7l9H1nqvY3McXO+yjbjbpA2fGFEzQX+Z+j6g+mzrlcYnPB0Bjpc/sPGeOSCfTGmw+BjsxkkwQEvYarZriM1eCz9i7aYJ87OSFgDyo96Dj9ke7EHOESYV0LzxN2WLN/3pYGTdr3Dg18krupCP+X3FAdgd+WP9TXKEcMFHcVs6y7Lyzis52irgEpSysqoOImQ8hdyLs/0oamXCnePoAzzHeTlw8IpUvhpTSYYtWMtCv9velIvxHsnSu87k3HD1jsZ/SZtsrrUwEFMs/9Z8wZQ6q5/GAKo6LRsXxv+BXpXNwc6K3qqxMi5D4TD0p7GBOuz6xmpzScr9eQVn1OqoekDvkdzeXmkOZC01XrJJfD1GA8fjUk7M2OTWeINNSNFiL3ovgz6cMDSuJq39vMPeH2CBW9ozjtMmODlmwEot4FHu3mq5eoajrJnpmpi7oJ6ks8icafs3GTIJDg6sAF3M3oxMXfVVKEUaqvYihq0u9T1qBjU4/raTWjUj2/DhVDNa0gX6OE/xUIdMZMqeeVHndFqb1lkORiPoXcTrD734HNvrmW73rI5TBWMK86RQ=="]
for given_key in given_key_list:
tested_result = []
for key in keysplit(given_key):
tested_result.append(key)
self.assertEqual(tested_result, expected_result)
if __name__ == '__main__':
unittest.main()
...@@ -51,29 +51,29 @@ pycrypto = 2.6.1 ...@@ -51,29 +51,29 @@ pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 2.8 slapos.recipe.template = 2.8
slapos.toolbox = 0.61 slapos.toolbox = 0.62
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
GitPython = 2.0.8 GitPython = 2.0.8
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
feedparser = 5.2.1 feedparser = 5.2.1
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
lockfile = 0.12.2 lockfile = 0.12.2
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
paramiko = 2.0.1 paramiko = 2.0.1
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
rpdb = 0.1.5 rpdb = 0.1.5
...@@ -20,6 +20,9 @@ extends = ...@@ -20,6 +20,9 @@ extends =
# Monitoring stack (keep on bottom) # Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
extensions -=
buildout-versions
parts += parts +=
template template
template-apache-frontend template-apache-frontend
...@@ -66,7 +69,7 @@ mode = 0644 ...@@ -66,7 +69,7 @@ mode = 0644
[template-slave-list] [template-slave-list]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum = 6828096d9ec4333b8c72a2e2ab768ea0 md5sum = 377aa36899c963ac65558b4888f8264b
mode = 640 mode = 640
[template-slave-configuration] [template-slave-configuration]
......
...@@ -49,6 +49,12 @@ ...@@ -49,6 +49,12 @@
"type": "string", "type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$" "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
}, },
"nginx-domain": {
"title": "Nginx Domain",
"description": "Base Domain for create subdomains (ie.: example2.com) for websocket, notebook and eventsource.",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"-frontend-quantity": { "-frontend-quantity": {
"title": "Frontend Replication Quantity", "title": "Frontend Replication Quantity",
"description": "Quantity of Frontends Replicate.", "description": "Quantity of Frontends Replicate.",
......
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
extends = common.cfg extends = common.cfg
[versions] [versions]
setuptools = 28.8.0
zc.buildout = 2.5.2+slapos005
zc.recipe.egg = 2.0.3+slapos002
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
apache-libcloud = 0.19.0 apache-libcloud = 0.19.0
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
...@@ -11,7 +15,7 @@ plone.recipe.command = 1.1 ...@@ -11,7 +15,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.template = 2.8 slapos.recipe.template = 2.8
slapos.toolbox = 0.61 slapos.toolbox = 0.62
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
...@@ -30,29 +34,29 @@ erp5.util = 0.4.46 ...@@ -30,29 +34,29 @@ erp5.util = 0.4.46
passlib = 1.6.5 passlib = 1.6.5
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
GitPython = 2.0.8 GitPython = 2.0.8
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
feedparser = 5.2.1 feedparser = 5.2.1
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
lockfile = 0.12.2 lockfile = 0.12.2
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
paramiko = 2.0.1 paramiko = 2.0.1
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
pycurl = 7.43.0 pycurl = 7.43.0
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
rpdb = 0.1.5 rpdb = 0.1.5
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
"title": "Default", "title": "Default",
"software-type": "default", "software-type": "default",
"description": "Default", "description": "Default",
"request": "instance-slave-apache-input-schema.json", "request": "instance-apache-input-schema.json",
"response": "instance-output-schema.json", "response": "instance-output-schema.json",
"index": 3 "index": 3
}, },
...@@ -33,7 +33,7 @@ ...@@ -33,7 +33,7 @@
"description": "Default", "description": "Default",
"request": "instance-slave-apache-input-schema.json", "request": "instance-slave-apache-input-schema.json",
"response": "instance-output-schema.json", "response": "instance-output-schema.json",
"index": 3, "index": 4,
"shared": true "shared": true
} }
} }
......
...@@ -71,7 +71,12 @@ crl = {{ custom_ssl_directory }}/crl/ ...@@ -71,7 +71,12 @@ crl = {{ custom_ssl_directory }}/crl/
{# Set slave domain if none was defined #} {# Set slave domain if none was defined #}
{% if slave_instance.get('custom_domain', None) == None -%} {% if slave_instance.get('custom_domain', None) == None -%}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (slave_instance.get('slave_reference').replace("-", "").lower(), slapparameter_dict.get('domain'))) -%} {% set domain_prefix = slave_instance.get('slave_reference').replace("-", "").lower() -%}
{% if slave_type in NGINX_TYPE_LIST -%}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('nginx-domain', slapparameter_dict.get('domain')))) -%}
{% else -%}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) -%}
{% endif -%}
{% endif -%} {% endif -%}
{% set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') -%} {% set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') -%}
......
...@@ -109,9 +109,6 @@ install = ...@@ -109,9 +109,6 @@ install =
extends = /mnt/slapos/software/hellorina/software.cfg extends = /mnt/slapos/software/hellorina/software.cfg
develop = /mnt/slapos develop = /mnt/slapos
download-cache = /mnt/download-cache download-cache = /mnt/download-cache
# Required for recent buildout.
extensions -=
buildout-versions
[slapos-cookbook-develop] [slapos-cookbook-develop]
recipe = recipe =
setup = setup =
...@@ -121,10 +118,8 @@ install = ...@@ -121,10 +118,8 @@ install =
[versions] [versions]
slapos.cookbook = slapos.cookbook =
EOF EOF
:||{ # https://github.com/pypa/setuptools/pull/846
python -S /mnt/buildout/bin/buildout bootstrap python -S /mnt/buildout/bin/buildout bootstrap
MAKEFLAGS=-j${:smp} bin/buildout MAKEFLAGS=-j${:smp} bin/buildout
}
arch=`dpkg-architecture -qDEB_HOST_ARCH` arch=`dpkg-architecture -qDEB_HOST_ARCH`
sudo sh -c "/mnt/slapos.package/playbook/roles/rina/gen-ipcm-conf sudo sh -c "/mnt/slapos.package/playbook/roles/rina/gen-ipcm-conf
......
...@@ -77,8 +77,10 @@ packages += ...@@ -77,8 +77,10 @@ packages +=
git ca-certificates python git ca-certificates python
# biggest and common build-deps for RINA # biggest and common build-deps for RINA
dh-autoreconf pkg-config doxygen maven xmlto dh-autoreconf pkg-config doxygen maven xmlto
# hellorina (shouldn't parts like lxml-python depend on the python of the SR?)
python-dev
[versions] [versions]
setuptools = 28.8.0 setuptools = 30.1.0
zc.buildout = 2.5.2+slapos005 zc.buildout = 2.5.2+slapos005
zc.recipe.egg = 2.0.3+slapos002 zc.recipe.egg = 2.0.3+slapos002
#!{{ bash.location }}/bin/bash -e
# export gitlab site via `gitlab-backup pull` to a "for-export" git-backup repository
# gitlab-export <exportto-repo>
#
# NOTE gitlab-backup, gitlab-rails, ... all have to be on $PATH.
# which site to export is determined by which gitlab-rails is on $PATH.
if [ "$#" -ne 1 ]; then
echo "Usage: gitlab-export <exportto-repo>" 1>&2
exit 1
fi
exportto_repo="$1"
# create / setup export repository if it does not exist yet
mkdir -p "$exportto_repo"
cd "$exportto_repo"
# verify we are in a git repository
if ! git rev-parse --is-inside-git-dir ; then
git init --bare
fi
exec gitlab-backup pull
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
# #
# TODO better autogenerate from ^^^ (?) # TODO better autogenerate from ^^^ (?)
# #
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
[gitlab-parameters] [gitlab-parameters]
configuration.external_url = http://lab.example.com configuration.external_url = http://lab.example.com
......
# GitLab instance + site export support
[buildout]
extends = {{ instance_gitlab_cfg }}
# TODO + ${pbsready-export:output}
parts +=
cron-entry-gitlab-backup
# -export specific instance parameters
[instance-parameter]
# cron frequency for gitlab backup (default: every 4h)
configuration.backup_frequency = 0 */4 * * *
[gitlab-backup-directory]
recipe = slapos.cookbook:mkdirectory
srv = ${buildout:directory}/srv
backup = ${:srv}/backup
backup-gitlab.git = ${:backup}/backup-gitlab.git
var = ${buildout:directory}/var
pid = ${:var}/pid
# instance exporter script
[exporter]
recipe = slapos.cookbook:wrapper
wrapper-path = ${buildout:directory}/bin/gitlab-exporter
command-line = {{ xnice_repository_location }}/bin/xnice {{ gitlab_export }} ${gitlab-backup-directory:backup-gitlab.git}
pidfile = ${gitlab-backup-directory:pid}/gitlab-exporter.pid
environment =
# XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
# `which` command in gitlab-backup, `chrt` in xnice, ...
# and `/bin` for `sed` command in gitlab-backup restore
PATH=/bin:/usr/bin:${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin
[cron-entry-gitlab-backup]
<= cron-entry
# run backup script on a regular basis (given as instance parameter)
frequency = ${instance-parameter:configuration.backup_frequency}
command = ${exporter:wrapper-path}
...@@ -537,7 +537,7 @@ depend = ...@@ -537,7 +537,7 @@ depend =
[promise-gitlab-workhorse] [promise-gitlab-workhorse]
<= promise-byurl <= promise-byurl
url = --unix-socket ${gitlab-workhorse:socket} http:/static.css url = --unix-socket ${gitlab-workhorse:socket} http://localhost/static.css
# gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us # gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us
...@@ -579,7 +579,7 @@ depend = ...@@ -579,7 +579,7 @@ depend =
[promise-unicorn] [promise-unicorn]
<= promise-byurl <= promise-byurl
url = --unix-socket ${unicorn:socket} http:/ url = --unix-socket ${unicorn:socket} http://localhost/
[promise-rakebase] [promise-rakebase]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -627,7 +627,7 @@ log = ${sidekiq-dir:log} ...@@ -627,7 +627,7 @@ log = ${sidekiq-dir:log}
# NOTE see queue list here: # NOTE see queue list here:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
[service-sidekiq] [service-sidekiq]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/sidekiq wrapper-path = ${directory:service}/sidekiq
......
...@@ -10,15 +10,17 @@ offline = true ...@@ -10,15 +10,17 @@ offline = true
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = $${instance-gitlab.cfg:rendered} gitlab = $${instance-gitlab.cfg:rendered}
# TODO -export, -import, -pull-backup gitlab-export = $${instance-gitlab-export.cfg:rendered}
default = $${:gitlab}
# TODO -import, -pull-backup
[instance-gitlab.cfg] # macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
mode = 0644 mode = 0644
template= ${instance-gitlab.cfg.in:target} rendered= $${buildout:directory}/$${:_buildout_section_name_}
rendered= $${buildout:directory}/instance-gitlab.cfg
context = context =
import os os import os os
import pwd pwd import pwd pwd
...@@ -33,12 +35,16 @@ context = ...@@ -33,12 +35,16 @@ context =
raw bash_bin ${bash:location}/bin/bash raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location} raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle} raw bundler_4gitlab ${bundler-4gitlab:bundle}
raw coreutils_location ${coreutils:location}
raw curl_bin ${curl:location}/bin/curl raw curl_bin ${curl:location}/bin/curl
raw dcron_bin ${dcron-output:crond} raw dcron_bin ${dcron-output:crond}
raw git ${git:location}/bin/git raw git ${git:location}/bin/git
raw git_location ${git:location} raw git_location ${git:location}
raw gitlab_export ${gitlab-export:rendered}
raw gitlab_workhorse ${gitlab-workhorse:location}/gitlab-workhorse raw gitlab_workhorse ${gitlab-workhorse:location}/gitlab-workhorse
raw gopath_bin ${gopath:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location}
raw gzip_bin ${gzip:location}/bin/gzip raw gzip_bin ${gzip:location}/bin/gzip
raw gzip_location ${gzip:location} raw gzip_location ${gzip:location}
raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate
...@@ -48,7 +54,9 @@ context = ...@@ -48,7 +54,9 @@ context =
raw postgresql_location ${postgresql92:location} raw postgresql_location ${postgresql92:location}
raw redis_binprefix ${redis28:location}/bin raw redis_binprefix ${redis28:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location} raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location}
raw watcher_sigkill ${watcher-sigkill:rendered} raw watcher_sigkill ${watcher-sigkill:rendered}
raw xnice_repository_location ${xnice-repository:location}
# config files # config files
raw database_yml_in ${database.yml.in:target} raw database_yml_in ${database.yml.in:target}
...@@ -64,3 +72,16 @@ context = ...@@ -64,3 +72,16 @@ context =
raw resque_yml_in ${resque.yml.in:target} raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target} raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target} raw unicorn_rb_in ${unicorn.rb.in:target}
$${:context-extra}
context-extra =
[instance-gitlab.cfg]
<= instance-cfg
template= ${instance-gitlab.cfg.in:target}
[instance-gitlab-export.cfg]
<= instance-cfg
template= ${instance-gitlab-export.cfg.in:target}
context-extra =
raw instance_gitlab_cfg $${instance-gitlab.cfg:rendered}
...@@ -14,9 +14,12 @@ extends = ...@@ -14,9 +14,12 @@ extends =
../../component/nginx/buildout.cfg ../../component/nginx/buildout.cfg
# for instance # for instance
../../component/coreutils/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/grep/buildout.cfg
../../component/bzip2/buildout.cfg ../../component/bzip2/buildout.cfg
../../component/curl/buildout.cfg ../../component/curl/buildout.cfg
../../component/tar/buildout.cfg
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
...@@ -36,20 +39,29 @@ parts = ...@@ -36,20 +39,29 @@ parts =
gitlab-shell/vendor gitlab-shell/vendor
gitlab/vendor/bundle gitlab/vendor/bundle
gitlab-workhorse gitlab-workhorse
git-backup
# for instance # for instance
instance.cfg instance.cfg
# TODO stop using slapos.cookbook-develop once 571d6514 gets into released
# slapos.cookbook egg
slapos-cookbook-develop
slapos-cookbook slapos-cookbook
eggs eggs
bash bash
curl curl
watcher-sigkill watcher-sigkill
gitlab-export
gzip gzip
dcron-output dcron-output
logrotate logrotate
[slapos.cookbook-repository]
revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
############################ ############################
# Software compilation # # Software compilation #
...@@ -113,15 +125,15 @@ git-executable = ${git:location}/bin/git ...@@ -113,15 +125,15 @@ git-executable = ${git:location}/bin/git
<= git-repository <= git-repository
#repository = https://gitlab.com/gitlab-org/gitlab-ce.git #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 8.7.X + NXD patches: # 8.8.X + NXD patches:
revision = v8.7.9-10-g7728df66b90483dac467df95948a532783e782c8 revision = v8.8.9-10-g967afbdc3a2d2bcc58ed31ca50d5293fa2a2e9ed
location = ${buildout:parts-directory}/gitlab location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository] [gitlab-shell-repository]
<= git-repository <= git-repository
#repository = https://gitlab.com/gitlab-org/gitlab-shell.git #repository = https://gitlab.com/gitlab-org/gitlab-shell.git
repository = https://lab.nexedi.com/nexedi/gitlab-shell.git repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
# gitlab 8.7 wants gitlab-shell 2.7.2 # gitlab 8.8 wants gitlab-shell 2.7.2
# 2.7.2 + NXD patches # 2.7.2 + NXD patches
revision = v2.6.10-50-gfbca95be784816349abc5930324659151eca50d1 revision = v2.6.10-50-gfbca95be784816349abc5930324659151eca50d1
location = ${buildout:parts-directory}/gitlab-shell location = ${buildout:parts-directory}/gitlab-shell
...@@ -130,7 +142,7 @@ location = ${buildout:parts-directory}/gitlab-shell ...@@ -130,7 +142,7 @@ location = ${buildout:parts-directory}/gitlab-shell
<= git-repository <= git-repository
#repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git #repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git
repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
# gitlab 8.7 wants gitlab-workhorse 0.7.1 # gitlab 8.8 wants gitlab-workhorse 0.7.1
# 0.7.1 + NXD patches # 0.7.1 + NXD patches
revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48 revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
location = ${buildout:parts-directory}/gitlab-workhorse location = ${buildout:parts-directory}/gitlab-workhorse
...@@ -151,6 +163,52 @@ make-binary = ...@@ -151,6 +163,52 @@ make-binary =
make-targets= cd ${:path} && make-targets= cd ${:path} &&
${:bundle} install --deployment --without development test mysql kerberos ${:bundle} install --deployment --without development test mysql kerberos
# directories and repositories required by gitlab-backup
[gopath]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
[go-git-repository]
<= git-repository
repository = https://${:go.importpath}.git
location = ${gopath:src}/${:go.importpath}
[git2go-repository]
<= go-git-repository
go.importpath = github.com/libgit2/git2go
# branch 'next' is required by git-backup
revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
[git-backup-repository]
<= go-git-repository
go.importpath = lab.nexedi.com/kirr/git-backup
revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
[git-backup]
# install git2go, git-backup, gitlab-backup in <gopath>/bin
recipe = slapos.recipe.cmmi
path = ${git-backup-repository:location}
configure-command = :
make-binary =
make-targets= cd ${git2go-repository:location}
&& git submodule update --init
&& make install
&& install -d ${gopath:bin}
&& go install ${git-backup-repository:go.importpath}
&& cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin}
environment =
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gopath:directory}
[xnice-repository]
# to get kirr's misc repo containing xnice script for executing processes
# with lower priority (used for backup script inside the cron)
<= git-repository
repository = https://lab.nexedi.com/kirr/misc.git
revision = 4073572ea700bf1b115f3a135aebebe5b3b824e4
location = ${buildout:parts-directory}/misc
# build needed-by-gitlab-shell gems via bundler # build needed-by-gitlab-shell gems via bundler
# ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error # ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error
...@@ -194,17 +252,26 @@ eggs = ...@@ -194,17 +252,26 @@ eggs =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
md5sum = ef85f02c4f6070c586d773b859a2f4e2 md5sum = b99a99b161c0b292845002fc3fee50cd
[watcher-sigkill] # macro: download a shell script and put it rendered into <software>/bin/
[binsh]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:bin-directory}/${:_buildout_section_name_} rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755 mode = 0755
md5sum = 2986dcb006dc9e8508ff81f646656131
context = context =
section bash bash section bash bash
[watcher-sigkill]
<= binsh
md5sum = 2986dcb006dc9e8508ff81f646656131
[gitlab-export]
<= binsh
md5sum = a7b32680e80f34276f0a32a5e22dad50
# macro: download a file named as section name # macro: download a file named as section name
# #
...@@ -232,7 +299,7 @@ md5sum = eb1230fee50067924ba89f4dc6e82fa9 ...@@ -232,7 +299,7 @@ md5sum = eb1230fee50067924ba89f4dc6e82fa9
[gitlab-parameters.cfg] [gitlab-parameters.cfg]
<= download-file <= download-file
md5sum = 3edd435a984b51b94539ea1d4f1b3994 md5sum = a74670934ec0190cc212d1f3468c11ed
[gitlab-shell-config.yml.in] [gitlab-shell-config.yml.in]
<= download-template <= download-template
...@@ -244,11 +311,15 @@ md5sum = a9cb347f60aad3465932fd36cd4fe25d ...@@ -244,11 +311,15 @@ md5sum = a9cb347f60aad3465932fd36cd4fe25d
[gitlab.yml.in] [gitlab.yml.in]
<= download-template <= download-template
md5sum = 735a78d0733fd6617d3b5f3d91bfae8c md5sum = 176939a6428a7aca4767a36421b0af2b
[instance-gitlab.cfg.in] [instance-gitlab.cfg.in]
<= download-file <= download-file
md5sum = ba83f01fd7a313b984766c50d6e48e54 md5sum = 89914e4a225f6cdebfa196d46359f6f2
[instance-gitlab-export.cfg.in]
<= download-file
md5sum = 00bd4ddf75a40a9d18d8243289f68ee9
[macrolib.cfg.in] [macrolib.cfg.in]
<= download-file <= download-file
...@@ -260,11 +331,11 @@ md5sum = a11b50d2ff2b1fa842ba4aa20041e2fe ...@@ -260,11 +331,11 @@ md5sum = a11b50d2ff2b1fa842ba4aa20041e2fe
[nginx.conf.in] [nginx.conf.in]
<= download-template <= download-template
md5sum = 7da68dba86fff79eb93c27aa1aaf1055 md5sum = 1374f38ab6f295b850d45ea0019ec05d
[rack_attack.rb.in] [rack_attack.rb.in]
<= download-template <= download-template
md5sum = bc1a7c1e83b7329d97bff6724f2bec3e md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
[resque.yml.in] [resque.yml.in]
<= download-template <= download-template
...@@ -272,7 +343,7 @@ md5sum = 7c89a730889e3224548d9abe51a2d719 ...@@ -272,7 +343,7 @@ md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in] [smtp_settings.rb.in]
<= download-template <= download-template
md5sum = d66a424516ffacea34303e2f512a7d94 md5sum = c67ea492e17f774d0e18f1217338a55f
[unicorn.rb.in] [unicorn.rb.in]
<= download-template <= download-template
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# see: # see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %} {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %}
...@@ -86,6 +86,7 @@ production: &base ...@@ -86,6 +86,7 @@ production: &base
wiki: {{ cfg('default_projects_features.wiki') }} wiki: {{ cfg('default_projects_features.wiki') }}
snippets: {{ cfg('default_projects_features.snippets') }} snippets: {{ cfg('default_projects_features.snippets') }}
builds: false {# builds not supported yet <%= @gitlab_default_projects_features_builds %> #} builds: false {# builds not supported yet <%= @gitlab_default_projects_features_builds %> #}
{# container_registry: <%= @gitlab_default_projects_features_container_registry %> #}
## Webhook settings ## Webhook settings
# Number of seconds to wait for HTTP response after sending webhook HTTP POST request (default: 10) # Number of seconds to wait for HTTP response after sending webhook HTTP POST request (default: 10)
...@@ -145,6 +146,18 @@ production: &base ...@@ -145,6 +146,18 @@ production: &base
storage_path: <%= @lfs_storage_path %> storage_path: <%= @lfs_storage_path %>
#} #}
{# we do not support container registry
## Container Registry
registry:
enabled: <%= @registry_enabled %>
host: <%= @registry_host %>
port: <%= @registry_port %>
api_url: <%= @registry_api_url %> # internal address to the registry, will be used by GitLab to directly communicate with API
path: <%= @registry_path %>
key: <%= @registry_key_path %>
issuer: <%= @registry_issuer %>
#}
{# we do not support Pages {# we do not support Pages
## GitLab Pages (EE only) ## GitLab Pages (EE only)
pages: pages:
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# see: # see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %} {% from 'macrolib.cfg.in' import cfg with context %}
...@@ -50,9 +50,11 @@ http { ...@@ -50,9 +50,11 @@ http {
include {{ nginx_gitlab_http_conf }}; include {{ nginx_gitlab_http_conf }};
{# we don't need: ci, pages, mattermost {# we don't need: ci, pages, mattermost, registry
include <%= @gitlab_ci_http_config %> include <%= @gitlab_ci_http_config %>
include <%= @gitlab_pages_http_config %>; include <%= @gitlab_pages_http_config %>;
include <%= @gitlab_mattermost_http_config %> include <%= @gitlab_mattermost_http_config %>
include <%= @gitlab_registry_http_config %>;
#} #}
} }
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# see: # see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %} {% from 'macrolib.cfg.in' import cfg with context %}
...@@ -24,8 +24,9 @@ paths_to_be_protected = [ ...@@ -24,8 +24,9 @@ paths_to_be_protected = [
# Create one big regular expression that matches strings starting with any of # Create one big regular expression that matches strings starting with any of
# the paths_to_be_protected. # the paths_to_be_protected.
paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ }) paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
unless Rails.env.test? unless Rails.env.test? || !rack_attack_enabled
Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req| Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
if req.post? && req.path =~ paths_regex if req.post? && req.path =~ paths_regex
req.ip req.ip
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# see: # see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7) # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_bool with context %} {% from 'macrolib.cfg.in' import cfg, cfg_bool with context %}
......
...@@ -6,6 +6,10 @@ parts = ...@@ -6,6 +6,10 @@ parts =
slapos-cookbook slapos-cookbook
template template
# Required for recent buildout.
extensions -=
buildout-versions
[template] [template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
...@@ -40,3 +44,8 @@ recipe = slapos.recipe.build:download ...@@ -40,3 +44,8 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_} url = ${:_profile_base_location_}/${:_buildout_section_name_}
mode = 755 mode = 755
md5sum = 78b77a6bda9958f547f7d89b747731e3 md5sum = 78b77a6bda9958f547f7d89b747731e3
[versions]
setuptools = 30.1.0
zc.buildout = 2.5.2+slapos005
zc.recipe.egg = 2.0.3+slapos002
...@@ -5,6 +5,8 @@ parts = ...@@ -5,6 +5,8 @@ parts =
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
extends = ${template-resilient-templates:output}
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = $${:test} default = $${:test}
...@@ -120,7 +122,7 @@ context = ...@@ -120,7 +122,7 @@ context =
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
raw kvm_template $${dynamic-template-kvm:rendered} raw kvm_template $${dynamic-template-kvm:rendered}
raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename} raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
raw pbsready_export_template ${pbsready-export:output} key pbsready_export_template template-pbsready-export:rendered
raw gzip_binary ${gzip:location}/bin/gzip raw gzip_binary ${gzip:location}/bin/gzip
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
mode = 0644 mode = 0644
......
...@@ -5,6 +5,6 @@ extends = development.cfg ...@@ -5,6 +5,6 @@ extends = development.cfg
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in
md5sum = 61414eefb6641f74d5f2b4ffc23af393 md5sum = 9e026495cbb0f5ea43adb2afb8574475
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
...@@ -5,7 +5,7 @@ extends = common.cfg ...@@ -5,7 +5,7 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1 websockify = 0.5.1
slapos.toolbox = 0.61 slapos.toolbox = 0.62
erp5.util = 0.4.46 erp5.util = 0.4.46
apache-libcloud = 1.1.0 apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
parts += parts +=
neo-admin-promise neo-admin-promise
logrotate-admin logrotate-admin
neoctl
[neo-admin] [neo-admin]
recipe = slapos.cookbook:neoppod.admin recipe = slapos.cookbook:neoppod.admin
...@@ -27,3 +28,15 @@ backup = ${logrotate:backup} ...@@ -27,3 +28,15 @@ backup = ${logrotate:backup}
name = neo-admin name = neo-admin
log = ${neo-admin:logfile} log = ${neo-admin:logfile}
post = {{ bin_directory }}/slapos-kill -n neoadmin -s RTMIN+1 ${:log} post = {{ bin_directory }}/slapos-kill -n neoadmin -s RTMIN+1 ${:log}
[neoctl]
recipe = slapos.cookbook:wrapper
command-line =
{{bin_directory}}/neoctl -a ${neo-admin:ip}:${neo-admin:port}
{%- if slapparameter_dict['ssl'] %}
--ca ${directory:etc}/ca.crt
--cert ${directory:etc}/neo.crt
--key ${directory:etc}/neo.key
{%- endif %}
parameters-extra = True
wrapper-path = ${directory:bin}/neoctl
...@@ -33,7 +33,7 @@ innodb_locks_unsafe_for_binlog = 1 ...@@ -33,7 +33,7 @@ innodb_locks_unsafe_for_binlog = 1
{{x}}sync_frm = 0 {{x}}sync_frm = 0
# Extra parameters. # Extra parameters.
{%- do extra_dict.setdefault('innodb_file_per_table', '0') %} {%- do extra_dict.setdefault('innodb_file_per_table', '1') %}
{%- for k, v in extra_dict.iteritems() %} {%- for k, v in extra_dict.iteritems() %}
{%- do assert('-' not in k) %} {%- do assert('-' not in k) %}
{{ k }} = {{ v }} {{ k }} = {{ v }}
......
...@@ -17,7 +17,7 @@ extends = ...@@ -17,7 +17,7 @@ extends =
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
parts = parts =
# keep neoppod first so that ZODB3 is built correctly, # keep neoppod first so that ZODB is built correctly,
# before any other section that would depend on it # before any other section that would depend on it
neoppod-develop neoppod-develop
neoppod neoppod
...@@ -40,9 +40,13 @@ environment = neoppod-setup-env ...@@ -40,9 +40,13 @@ environment = neoppod-setup-env
[neoppod] [neoppod]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests] eggs = neoppod[admin, ctl, master, storage-mysqldb]
${python-mysqlclient:egg} ${python-mysqlclient:egg}
ZODB3 psutil
BTrees
ZODB
zope.testing
zodbtools
[slapos-deps-eggs] [slapos-deps-eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -85,7 +89,7 @@ md5sum = f3259726bd5d824c569dc7db6b7d26a0 ...@@ -85,7 +89,7 @@ md5sum = f3259726bd5d824c569dc7db6b7d26a0
[instance-neo-admin] [instance-neo-admin]
<= download-base-neo <= download-base-neo
md5sum = f030a25d320f2edf0186b69bfa521228 md5sum = ec1e2d9d2a1da092621f1b01ce6af322
[instance-neo-master] [instance-neo-master]
<= download-base-neo <= download-base-neo
...@@ -98,13 +102,19 @@ md5sum = cd2a978a09c5686205592923866f6584 ...@@ -98,13 +102,19 @@ md5sum = cd2a978a09c5686205592923866f6584
[template-neo-my-cnf] [template-neo-my-cnf]
<= download-base-neo <= download-base-neo
url = ${:_profile_base_location_}/my.cnf.in url = ${:_profile_base_location_}/my.cnf.in
md5sum = 81ab5e842ecf8385b12d735585497cc8 md5sum = cfdd59d42ae540563a964ccaf8fec232
[versions] [versions]
slapos.recipe.template = 2.9 slapos.recipe.template = 2.9
ZODB3 = 3.10.7 ZODB = 4.4.4
# Required by slapos.toolbox = 0.61 BTrees = 4.3.1
slapos.toolbox = 0.61 persistent = 4.2.2
transaction = 1.7.0
zdaemon = 4.1.0
zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3
# Required by slapos.toolbox = 0.62
slapos.toolbox = 0.62
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
apache-libcloud = 1.2.1 apache-libcloud = 1.2.1
atomize = 0.2.0 atomize = 0.2.0
......
[buildout]
extends = software.cfg
[neoppod]
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
${python-mysqlclient:egg}
ZODB3
patch-binary = ${patch:location}/bin/patch
ZODB3-patch-options = -p1
ZODB3-patches =
${neoppod-repository:location}/ZODB3.patch
[versions]
ZODB3 = 3.10.7+SlapOSPatched001
transaction = 1.1.1
zdaemon = 2.0.7
[buildout]
extends = software.cfg
[neoppod]
eggs = neoppod
${python-mysqlclient:egg}
psutil
ZODB
zope.testing
[versions]
ZODB = 4.4.3
transaction =
zdaemon =
...@@ -33,19 +33,12 @@ recipe = zc.recipe.egg ...@@ -33,19 +33,12 @@ recipe = zc.recipe.egg
eggs = erp5.util eggs = erp5.util
interpreter = ${:_buildout_section_name_} interpreter = ${:_buildout_section_name_}
[neoppod]
patch-binary = ${patch:location}/bin/patch
ZODB3-patch-options = -p1
ZODB3-patches +=
${neoppod-repository:location}/ZODB3.patch
[versions] [versions]
ZODB3 = 3.10.7+SlapOSPatched001
erp5.util = 0.4.46 erp5.util = 0.4.46
# To match ERP5 # To match ERP5
transaction = 1.1.1
ZConfig = 2.9.3 ZConfig = 2.9.3
zc.lockfile = 1.0.2 zc.lockfile = 1.0.2
zdaemon = 2.0.7
zope.event = 3.5.2 zope.event = 3.5.2
zope.exceptions = 3.6.2
zope.testing = 3.9.7
## ##
...@@ -115,15 +115,15 @@ plone.recipe.command = 1.1 ...@@ -115,15 +115,15 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 2.7 slapos.recipe.template = 2.7
slapos.toolbox = 0.61 slapos.toolbox = 0.62
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
GitPython = 2.0.8 GitPython = 2.0.8
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
...@@ -131,11 +131,11 @@ atomize = 0.2.0 ...@@ -131,11 +131,11 @@ atomize = 0.2.0
backports.ssl-match-hostname = 3.4.0.2 backports.ssl-match-hostname = 3.4.0.2
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
lockfile = 0.12.2 lockfile = 0.12.2
# Required by: # Required by:
...@@ -143,10 +143,10 @@ lockfile = 0.12.2 ...@@ -143,10 +143,10 @@ lockfile = 0.12.2
miniupnpc = 1.9 miniupnpc = 1.9
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
paramiko = 2.0.1 paramiko = 2.0.1
# Required by: # Required by:
# slapos.toolbox = 0.61 # slapos.toolbox = 0.62
rpdb = 0.1.5 rpdb = 0.1.5
...@@ -60,7 +60,7 @@ mode = 0644 ...@@ -60,7 +60,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in output = ${buildout:directory}/template-runner.cfg.in
md5sum = 0b3561ee4ef8d687fa95f2915fe9923b md5sum = 07905ec872e84bc42c6ab089f5270ac2
mode = 0644 mode = 0644
[template-runner-import-script] [template-runner-import-script]
...@@ -68,7 +68,7 @@ recipe = hexagonit.recipe.download ...@@ -68,7 +68,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/template/runner-import.sh.jinja2 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
download-only = true download-only = true
md5sum = 275ae222cd9a560c08748d7502824885 md5sum = 5db39d7f56fd1f96ce6466e9055841a1
filename = runner-import.sh.jinja2 filename = runner-import.sh.jinja2
mode = 0644 mode = 0644
...@@ -84,7 +84,7 @@ recipe = hexagonit.recipe.download ...@@ -84,7 +84,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/template/runner-export.sh.jinja2 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
download-only = true download-only = true
md5sum = c835dff22a1e3de3dacb325acb691299 md5sum = e2d472ade09c11c70dbea080932e80bd
filename = runner-export.sh.jinja2 filename = runner-export.sh.jinja2
mode = 0644 mode = 0644
...@@ -125,7 +125,7 @@ recipe = hexagonit.recipe.download ...@@ -125,7 +125,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/httpd_conf.in url = ${:_profile_base_location_}/httpd_conf.in
download-only = true download-only = true
md5sum = 112cf8ada9e5c4172fa6fc464df0fd3a md5sum = 40825ef2d259ae3b86dfd2f28f597d3a
filename = httpd_conf.in filename = httpd_conf.in
mode = 0644 mode = 0644
......
...@@ -85,6 +85,12 @@ Alias /share {{ parameters.runner_home }} ...@@ -85,6 +85,12 @@ Alias /share {{ parameters.runner_home }}
AuthType Basic AuthType Basic
AuthName "Webrunner Dav" AuthName "Webrunner Dav"
AuthUserFile "{{ parameters.htpasswd_file }}" AuthUserFile "{{ parameters.htpasswd_file }}"
# Prevent using the web browser cache if requesting the same document
# from different domains or with different users
Header set Cache-Control "private, max-age=0, must-revalidate"
Header set Vary "Origin,Cookie,Authorization"
<LimitExcept OPTIONS> <LimitExcept OPTIONS>
Require valid-user Require valid-user
</LimitExcept> </LimitExcept>
......
...@@ -196,9 +196,11 @@ auto_deploy = True ...@@ -196,9 +196,11 @@ auto_deploy = True
[runtestsuite] [runtestsuite]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${buildout:directory}/bin/slaprunnertest arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-connection:key-file} --cert_file=$${slap-connection:cert-file} --computer_id=$${slap-connection:computer-id} --partition_id=$${slap-connection:partition-id}
command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments}
wrapper-path = $${directory:bin}/runTestSuite wrapper-path = $${directory:bin}/runTestSuite
environment = RUNNER_CONFIG=$${slapos-cfg:rendered} environment = RUNNER_CONFIG=$${slapos-cfg:rendered}
parameters-extra = true
# Deploy openssh-server # Deploy openssh-server
[runner-sshd-port] [runner-sshd-port]
......
...@@ -17,7 +17,7 @@ gunicorn = 19.4.5 ...@@ -17,7 +17,7 @@ gunicorn = 19.4.5
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 2.9 slapos.recipe.template = 2.9
slapos.toolbox = 0.61 slapos.toolbox = 0.62
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -61,3 +61,36 @@ if [ -d {{ directory['backup'] }}/runner/software ]; then ...@@ -61,3 +61,36 @@ if [ -d {{ directory['backup'] }}/runner/software ]; then
fi fi
cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
# Check that export didn't happen during backup of instances
tmp_backup_sum=$(mktemp -p {{ directory['tmp'] }})
tmp_filtered_signature=$(mktemp -p {{ directory['tmp'] }})
remove_tmp_files () {
rm $tmp_backup_sum
rm $tmp_filtered_signature
}
trap remove_tmp_files EXIT
cd $srv_directory
backup_directory_path=$(find . -path "./runner/instance/slappart*/srv/backup/*" -type f)
# If no backup found, it's over
if [ -z "$backup_directory_path" ]; then
exit 0
fi
sleep 5
sha256sum $backup_directory_path | LC_ALL=C sort -k 66 > $tmp_backup_sum
cat {{ directory['backup'] }}/backup.signature | egrep "instance/slappart.*/srv/backup/" > $tmp_filtered_signature
# If the diff fails, then the notifier will restart this script
diff_status=0
diff $tmp_backup_sum $tmp_filtered_signature || diff_status=1
if [ $diff_status -ne 0 ]; then
echo "ERROR: Some backups are not consistents, exporter should be re-run."
echo "Let's sleep 10 minutes, to let the backup end..."
sleep 10m
exit 1
fi
...@@ -15,6 +15,7 @@ ERROR_MESSAGE="" ...@@ -15,6 +15,7 @@ ERROR_MESSAGE=""
fail_with_exit_code () { fail_with_exit_code () {
echo 1 > $RESTORE_EXIT_CODE_FILE echo 1 > $RESTORE_EXIT_CODE_FILE
echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
exit 1
} }
trap fail_with_exit_code ERR trap fail_with_exit_code ERR
...@@ -55,18 +56,18 @@ cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }}; ...@@ -55,18 +56,18 @@ cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
# Invoke arbitrary script to perform specific restoration # Invoke arbitrary script to perform specific restoration
# procedure. # procedure.
RESTORE_EXIT_CODE=0
runner_import_restore=$srv_directory/runner-import-restore runner_import_restore=$srv_directory/runner-import-restore
if [ ! -e "$runner_import_restore" ]; then if [ ! -e "$runner_import_restore" ]; then
touch $runner_import_restore touch $runner_import_restore
chmod +x $runner_import_restore chmod +x $runner_import_restore
fi fi
log_message "Running $runner_import_restore..." log_message "Running $runner_import_restore..."
$srv_directory/runner-import-restore || RESTORE_EXIT_CODE=$? $srv_directory/runner-import-restore
# If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume # If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume
# that there is no instnace deployed on runner0 # that there is no instnace deployed on runner0
if [ ! -f "directory['etc']/.project" ] && [ ! -f "$srv_directory/runner/proxy.db" ]; then if [ ! -f "directory['etc']/.project" ] && [ ! -f "$srv_directory/runner/proxy.db" ]; then
log_message "No Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0 exit 0
fi fi
...@@ -121,8 +122,8 @@ for partition in $srv_directory/runner/instance/slappart*/ ...@@ -121,8 +122,8 @@ for partition in $srv_directory/runner/instance/slappart*/
do do
script=$partition/srv/runner-import-restore script=$partition/srv/runner-import-restore
if [ -e "$script" ]; then if [ -e "$script" ]; then
log_message "Running $script..." log_message "Running custom instance script : $script..."
$script || RESTORE_EXIT_CODE=$? $script
fi fi
done done
...@@ -132,5 +133,5 @@ $SQLITE3 $DATABASE "update partition11 set requested_state='started';" ...@@ -132,5 +133,5 @@ $SQLITE3 $DATABASE "update partition11 set requested_state='started';"
# Write exit code to an arbitrary file that will be checked by promise/monitor # Write exit code to an arbitrary file that will be checked by promise/monitor
log_message "Writing status file... End" log_message "Writing status file... End"
echo $RESTORE_EXIT_CODE > $RESTORE_EXIT_CODE_FILE echo 0 > $RESTORE_EXIT_CODE_FILE
exit $RESTORE_EXIT_CODE exit 0
...@@ -466,10 +466,9 @@ initialization = ...@@ -466,10 +466,9 @@ initialization =
[eggs] [eggs]
<= neoppod <= neoppod
eggs = eggs = ${neoppod:eggs}
${numpy:egg} ${numpy:egg}
${matplotlib:egg} ${matplotlib:egg}
${python-mysqlclient:egg}
${lxml-python:egg} ${lxml-python:egg}
${pandas:egg} ${pandas:egg}
${pillow-python:egg} ${pillow-python:egg}
...@@ -524,12 +523,12 @@ eggs = ...@@ -524,12 +523,12 @@ eggs =
pylint pylint
pytracemalloc pytracemalloc
neoppod[client]
# Zope # Zope
ZODB3
Zope2 Zope2
# Zope acquisition patch # Zope acquisition patch
Acquisition Acquisition
# for runzeo
ZEO
# Other Zope 2 packages # Other Zope 2 packages
Products.PluggableAuthService Products.PluggableAuthService
...@@ -683,6 +682,8 @@ PyXML = 0.8.5 ...@@ -683,6 +682,8 @@ PyXML = 0.8.5
Pympler = 0.4.3 Pympler = 0.4.3
StructuredText = 2.11.1 StructuredText = 2.11.1
WSGIUtils = 0.7 WSGIUtils = 0.7
ZEO = 4.2.1
ZODB3 = 3.11.0
Zope2 = 2.13.24 Zope2 = 2.13.24
astor = 0.5 astor = 0.5
# astroid 1.4.1 breaks testDynamicClassGeneration # astroid 1.4.1 breaks testDynamicClassGeneration
......
...@@ -96,7 +96,7 @@ recipe = slapos.recipe.template:jinja2 ...@@ -96,7 +96,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg rendered = ${buildout:directory}/template-monitor.cfg
md5sum = 266f23bc836a938ac69ce0bc674f34fe md5sum = 5b6b7a96bfa1e0ca83310caa48ed7b20
context = context =
key apache_location apache:location key apache_location apache:location
key gzip_location gzip:location key gzip_location gzip:location
...@@ -125,5 +125,5 @@ depends = ...@@ -125,5 +125,5 @@ depends =
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.61 slapos.toolbox = 0.62
...@@ -293,7 +293,7 @@ wrapper-path = ${directory:bin}/monitor-globalstate ...@@ -293,7 +293,7 @@ wrapper-path = ${directory:bin}/monitor-globalstate
[monitor-configurator-wrapper] [monitor-configurator-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# XXX - hard coded path # XXX - hard coded path
command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd' command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd' --monitor_https_cors {{ monitor_https_cors }}
wrapper-path = ${directory:bin}/monitor-configurator wrapper-path = ${directory:bin}/monitor-configurator
[monitor-globalstate-cron-entry] [monitor-globalstate-cron-entry]
...@@ -374,10 +374,12 @@ recipe = collective.recipe.template ...@@ -374,10 +374,12 @@ recipe = collective.recipe.template
file = ${monitor-conf-parameters:promise-output-file} file = ${monitor-conf-parameters:promise-output-file}
input = inline:#!{{ dash_executable_location }} input = inline:#!{{ dash_executable_location }}
pidfile=${monitor-conf-parameters:pid-file} pidfile=${monitor-conf-parameters:pid-file}
COMMAND=monitor.bootstrap
if [ -s $pidfile ]; then if [ -s $pidfile ]; then
for i in {1..20}; do COUNTER=0
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then # Wait until max 20 seconds, the limit promise timeout
while [ $COUNTER -lt 20 ]; do
if [ -n "$(ps -p $(cat $pidfile) -o pid=)" ]; then
((COUNTER=COUNTER+1))
sleep 0.5 sleep 0.5
else else
break break
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment