Commit 0569696f authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 2ec3b9f0 5691fd6f
......@@ -59,12 +59,16 @@ location = ${buildout:parts-directory}/ZEO4
git-executable = ${git:location}/bin/git
# ZEO5 is plain upstream egg
[ZEO5]
# ZEO6 and ZEO5 are plain upstream eggs
[ZEO6]
recipe = zc.recipe.egg:eggs
egg = ZEO
eggs = ${:egg}
egg-versions =
ZEO = 6.0.0
[ZEO5]
<= ZEO6
egg-versions =
ZEO = 5.4.0
trollius = 2.2.1
......
......@@ -9,16 +9,16 @@ extends =
parts = ZODB/scripts
# ZODB allows to use either ZODB4, ZODB4-wc2 or ZODB5.
# ZODB allows to use either ZODB4, ZODB4-wc2, ZODB5 or ZODB6.
# To select which version to use users should do:
#
# [ZODB]
# major = <ZODB-version-major>
#
# By default ZODB5 is used.
# By default latest is used: ZODB6 on python3 and ZODB5 on python2
[ZODB]
recipe = slapos.recipe.build
major = 5
major = 6
init =
# link/depend ZODB -> ZODB<X>
zodb_x = 'ZODB'+options['major']
......@@ -36,13 +36,15 @@ init =
import zc.buildout.easy_install
zc.buildout.easy_install.default_versions(versions)
[ZODB:python2]
major = 5
# ZODB/scripts installs scripts from ZODB
[ZODB/scripts]
recipe = zc.recipe.egg:scripts
eggs = ${ZODB:egg}
# ZODB4 and ZODB5 are plain upstream eggs
# ZODB4, ZODB5 and ZODB6 are plain upstream eggs
[_ZODB]
recipe = zc.recipe.egg:eggs
egg = ZODB
......@@ -57,10 +59,21 @@ egg-versions =
[ZODB5]
<= _ZODB
egg-versions =
ZODB = 5.8.1
transaction = 4.0.0
[ZODB5:python2]
<= _ZODB
egg-versions =
ZODB = 5.8.1
transaction = 3.0.1
[ZODB6]
<= _ZODB
egg-versions =
ZODB = 6.0.0
transaction = 4.0.0
# ZODB4-wc2 is ZODB4 version with patches for wendelin.core 2 to work correctly.
# The main change is backport of the way MVCC is handled by always calling
......@@ -92,13 +105,18 @@ egg = ${:_buildout_section_name_}
setup-eggs = ${python-cffi:egg}
# eggs that are common to ZODB4 and ZODB5.
# eggs that are common to ZODB4, ZODB5 and ZODB6.
[versions]
BTrees = 4.11.3
persistent = 4.9.3
zodbpickle = 2.6.0
BTrees = 5.1.0
persistent = 5.1.0
zodbpickle = 3.3.0
# Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
# for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_
# ZODB, persistent, BTrees and ZEO.
ZODB3 = 3.11.0
[versions:python2]
BTrees = 4.11.3
persistent = 4.9.3
zodbpickle = 2.6.0
......@@ -2,6 +2,7 @@
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../xz-utils/buildout.cfg
parts =
brctl
......@@ -12,7 +13,7 @@ recipe = slapos.recipe.cmmi
url = https://mirrors.edge.kernel.org/pub/linux/utils/net/bridge-utils/bridge-utils-1.7.1.tar.xz
md5sum = 3e1fee4dc22cac5457c2f6ffb990a518
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:%(PATH)s
PATH=${autoconf:location}/bin:${automake:location}/bin:${xz-utils:location}/bin:%(PATH)s
pre-configure =
aclocal &&
autoconf
......@@ -6,6 +6,7 @@ extends =
../pkgconfig/buildout.cfg
../json-c/buildout.cfg
../openssl/buildout.cfg
../xz-utils/buildout.cfg
[cryptsetup]
recipe = slapos.recipe.cmmi
......@@ -25,7 +26,7 @@ configure-options =
--disable-kernel_crypto
--disable-blkid
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${json-c:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
CFLAGS=-I${libuuid:location}/include -I${lvm2:location}/include -I${popt:location}/include -I${json-c:location}/include -I${openssl:location}/include
LDFLAGS=-L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -L${lvm2:location}/lib -Wl,-rpath=${lvm2:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib -L${json-c:location}/lib -Wl,-rpath=${json-c:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
From 3666a7afd46ea6d069606450c520b8b7e2b5fddf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Thu, 22 Feb 2024 23:33:41 +0900
Subject: [PATCH] Make dict views behave like their unrestricted versions
unlike the restricted versions, the unrestricted versions:
- are not iterators, they are views
- have a len
- are false when the mapping is empty, true otherwise
- are instances of collections.abc.MappingView
During this refactoring, also change `.items()` to validate
ach keys and values, like `.keys()` and `.values()` do.
---
CHANGES.rst | 7 ++++
src/AccessControl/ZopeGuards.py | 50 ++++++++++++++++++-----
src/AccessControl/tests/actual_python.py | 33 +++++++++++++++
src/AccessControl/tests/testZopeGuards.py | 34 +++++++++++----
4 files changed, 104 insertions(+), 20 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index f35a8d2..073b791 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -8,6 +8,13 @@ For changes before version 3.0, see ``HISTORY.rst``.
- Nothing changed yet.
+- Make dict views (`.keys()`, `.items()` and `.values()`) behave like their
+ unrestricted versions.
+ (`#147 <https://github.com/zopefoundation/AccessControl/pull/147>`_)
+
+- Make `.items()` validate each keys and values, like `.keys()` and
+ `.values()` do.
+
6.3 (2023-11-20)
----------------
diff --git a/src/AccessControl/ZopeGuards.py b/src/AccessControl/ZopeGuards.py
index 84c2e9e..bc24941 100644
--- a/src/AccessControl/ZopeGuards.py
+++ b/src/AccessControl/ZopeGuards.py
@@ -12,6 +12,7 @@
##############################################################################
+import collections.abc
import math
import random
import string
@@ -127,13 +128,18 @@ def guarded_pop(key, default=_marker):
return guarded_pop
-def get_iter(c, name):
- iter = getattr(c, name)
+def get_mapping_view(c, name):
- def guarded_iter():
- return SafeIter(iter(), c)
+ view_class = {
+ 'keys': SafeKeysView,
+ 'items': SafeItemsView,
+ 'values': SafeValuesView,
+ }
- return guarded_iter
+ def guarded_mapping_view():
+ return view_class[name](c)
+
+ return guarded_mapping_view
def get_list_pop(lst, name):
@@ -153,18 +159,15 @@ def guarded_pop(index=-1):
'copy': 1,
'fromkeys': 1,
'get': get_dict_get,
- 'items': 1,
+ 'items': get_mapping_view,
+ 'keys': get_mapping_view,
'pop': get_dict_pop,
'popitem': 1,
'setdefault': 1,
'update': 1,
+ 'values': get_mapping_view,
}
-_dict_white_list.update({
- 'keys': get_iter,
- 'values': get_iter,
-})
-
def _check_dict_access(name, value):
# Check whether value is a dict method
@@ -262,6 +265,31 @@ def __next__(self):
next = __next__
+class _SafeMappingView:
+ __allow_access_to_unprotected_subobjects__ = 1
+
+ def __iter__(self):
+ for e in super().__iter__():
+ guard(self._mapping, e)
+ yield e
+
+
+class SafeKeysView(_SafeMappingView, collections.abc.KeysView):
+ pass
+
+
+class SafeValuesView(_SafeMappingView, collections.abc.ValuesView):
+ pass
+
+
+class SafeItemsView(_SafeMappingView, collections.abc.ItemsView):
+ def __iter__(self):
+ for k, v in super().__iter__():
+ guard(self._mapping, k)
+ guard(self._mapping, v)
+ yield k, v
+
+
class NullIter(SafeIter):
def __init__(self, ob):
self._iter = ob
diff --git a/src/AccessControl/tests/actual_python.py b/src/AccessControl/tests/actual_python.py
index 3405b8e..866a480 100644
--- a/src/AccessControl/tests/actual_python.py
+++ b/src/AccessControl/tests/actual_python.py
@@ -123,6 +123,39 @@ def f7():
access = getattr(d, meth)
result = sorted(access())
assert result == expected[kind], (meth, kind, result, expected[kind])
+ assert len(access()) == len(expected[kind]), (meth, kind, "len")
+ iter_ = access() # iterate twice on the same view
+ assert list(iter_) == list(iter_)
+
+ assert sorted([k for k in getattr(d, meth)()]) == expected[kind]
+ assert sorted(k for k in getattr(d, meth)()) == expected[kind]
+ assert {k: v for k, v in d.items()} == d
+
+ assert 1 in d
+ assert 1 in d.keys()
+ assert 2 in d.values()
+ assert (1, 2) in d.items()
+
+ assert d
+ assert d.keys()
+ assert d.values()
+ assert d.items()
+
+ empty_d = {}
+ assert not empty_d
+ assert not empty_d.keys()
+ assert not empty_d.values()
+ assert not empty_d.items()
+
+ smaller_d = {1: 2}
+ for m, _ in methods:
+ assert getattr(d, m)() != getattr(smaller_d, m)()
+ assert not getattr(d, m)() == getattr(smaller_d, m)()
+ if m != 'values':
+ assert getattr(d, m)() > getattr(smaller_d, m)()
+ assert getattr(d, m)() >= getattr(smaller_d, m)()
+ assert getattr(smaller_d, m)() < getattr(d, m)()
+ assert getattr(smaller_d, m)() <= getattr(d, m)()
f7()
diff --git a/src/AccessControl/tests/testZopeGuards.py b/src/AccessControl/tests/testZopeGuards.py
index 533bfa2..50eeca9 100644
--- a/src/AccessControl/tests/testZopeGuards.py
+++ b/src/AccessControl/tests/testZopeGuards.py
@@ -258,23 +258,40 @@ def test_pop_validates(self):
self.assertTrue(sm.calls)
def test_keys_empty(self):
- from AccessControl.ZopeGuards import get_iter
- keys = get_iter({}, 'keys')
+ from AccessControl.ZopeGuards import get_mapping_view
+ keys = get_mapping_view({}, 'keys')
self.assertEqual(list(keys()), [])
+ def test_kvi_len(self):
+ from AccessControl.ZopeGuards import get_mapping_view
+ for attr in ("keys", "values", "items"):
+ with self.subTest(attr):
+ view = get_mapping_view({'a': 1}, attr)
+ self.assertEqual(len(view()), 1)
+
def test_keys_validates(self):
sm = SecurityManager()
old = self.setSecurityManager(sm)
keys = guarded_getattr({GuardTestCase: 1}, 'keys')
try:
- next(keys())
+ next(iter(keys()))
finally:
self.setSecurityManager(old)
self.assertTrue(sm.calls)
+ def test_items_validates(self):
+ sm = SecurityManager()
+ old = self.setSecurityManager(sm)
+ items = guarded_getattr({GuardTestCase: GuardTestCase}, 'items')
+ try:
+ next(iter(items()))
+ finally:
+ self.setSecurityManager(old)
+ self.assertEqual(len(sm.calls), 2)
+
def test_values_empty(self):
- from AccessControl.ZopeGuards import get_iter
- values = get_iter({}, 'values')
+ from AccessControl.ZopeGuards import get_mapping_view
+ values = get_mapping_view({}, 'values')
self.assertEqual(list(values()), [])
def test_values_validates(self):
@@ -282,18 +299,17 @@ def test_values_validates(self):
old = self.setSecurityManager(sm)
values = guarded_getattr({GuardTestCase: 1}, 'values')
try:
- next(values())
+ next(iter(values()))
finally:
self.setSecurityManager(old)
self.assertTrue(sm.calls)
def test_kvi_iteration(self):
- from AccessControl.ZopeGuards import SafeIter
d = dict(a=1, b=2)
for attr in ("keys", "values", "items"):
v = getattr(d, attr)()
- si = SafeIter(v)
- self.assertEqual(next(si), next(iter(v)))
+ si = guarded_getattr(d, attr)()
+ self.assertEqual(next(iter(si)), next(iter(v)))
class TestListGuards(GuardTestCase):
From 77f86b50f097dcf364e0d140e45593bf001d46bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Fri, 1 Mar 2024 09:49:17 +0900
Subject: [PATCH] set metadata in setup.py for compatibility with old slapos
buildout
---
setup.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/setup.py b/setup.py
index 1bf0bcff5..a93fe7b22 100755
--- a/setup.py
+++ b/setup.py
@@ -987,6 +987,11 @@ ext_modules = [
try:
setup(
+ name='pillow',
+ version='10.2.0',
+ packages=["PIL"],
+ include_package_data=True,
+ package_dir={"": "src"},
cmdclass={"build_ext": pil_build_ext},
ext_modules=ext_modules,
zip_safe=not (debug_build() or PLATFORM_MINGW),
--
2.42.0
From c233d7278ae7089ba2ad32b8a178a3793273a47d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Sat, 1 Jun 2024 14:58:23 +0900
Subject: [PATCH] import from zope.lifecycleevent.interfaces to prevent
DeprecationWarnings
partial backport from https://github.com/zopefoundation/Products.CMFCore/pull/125
---
src/Products/CMFCore/CMFCatalogAware.py | 4 ++--
src/Products/CMFCore/CachingPolicyManager.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/Products/CMFCore/CMFCatalogAware.py b/src/Products/CMFCore/CMFCatalogAware.py
index a574660..502161f 100644
--- a/src/Products/CMFCore/CMFCatalogAware.py
+++ b/src/Products/CMFCore/CMFCatalogAware.py
@@ -25,11 +25,11 @@ from OFS.interfaces import IObjectClonedEvent
from OFS.interfaces import IObjectWillBeMovedEvent
from zope.component import queryUtility
from zope.component import subscribers
-from zope.container.interfaces import IObjectAddedEvent
-from zope.container.interfaces import IObjectMovedEvent
from zope.interface import implementer
+from zope.lifecycleevent.interfaces import IObjectAddedEvent
from zope.lifecycleevent.interfaces import IObjectCopiedEvent
from zope.lifecycleevent.interfaces import IObjectCreatedEvent
+from zope.lifecycleevent.interfaces import IObjectMovedEvent
from .interfaces import ICallableOpaqueItem
from .interfaces import ICatalogAware
diff --git a/src/Products/CMFCore/CachingPolicyManager.py b/src/Products/CMFCore/CachingPolicyManager.py
index 3722b7f..65a079d 100644
--- a/src/Products/CMFCore/CachingPolicyManager.py
+++ b/src/Products/CMFCore/CachingPolicyManager.py
@@ -27,9 +27,9 @@ from Persistence import PersistentMapping
from Products.PageTemplates.Expressions import SecureModuleImporter
from Products.PageTemplates.Expressions import getEngine
from zope.component import getUtility
-from zope.container.interfaces import IObjectMovedEvent
from zope.datetime import rfc1123_date
from zope.interface import implementer
+from zope.lifecycleevent.interfaces import IObjectMovedEvent
from .Expression import Expression
from .interfaces import ICachingPolicy
--
2.42.0
From 2a248ffc584082122776f3b51f5fdaf90c5e2905 Mon Sep 17 00:00:00 2001
From: Maurits van Rees <maurits@vanrees.org>
Date: Tue, 23 Apr 2024 22:22:53 +0200
Subject: [PATCH] Use the new resources keyword of registerClass if available.
This avoids a deprecation warning for using a non callable constructor in Zope higher than 5.9.
See report in https://github.com/zopefoundation/Zope/issues/1202
---
CHANGES.rst | 6 ++++-
src/Products/CMFCore/utils.py | 51 ++++++++++++++++++++++++-----------
2 files changed, 40 insertions(+), 17 deletions(-)
diff --git a/src/Products/CMFCore/utils.py b/src/Products/CMFCore/utils.py
index 2f99d6d..565914d 100644
--- a/src/Products/CMFCore/utils.py
+++ b/src/Products/CMFCore/utils.py
@@ -599,13 +599,20 @@ def initialize(self, context):
# Add only one meta type to the folder add list.
productObject = context._ProductContext__prod
self.product_name = productObject.id
- context.registerClass(
- meta_type=self.meta_type,
- # This is a little sneaky: we add self to the
- # FactoryDispatcher under the name "toolinit".
- # manage_addTool() can then grab it.
- constructors=(manage_addToolForm, manage_addTool, self),
- icon=self.icon)
+ # We add self to the FactoryDispatcher under the name 'toolinit'.
+ # manage_addContentType() can then grab it.
+ try:
+ context.registerClass(
+ meta_type=self.meta_type,
+ constructors=(manage_addToolForm, manage_addTool),
+ resources=(self, ),
+ icon=self.icon)
+ except TypeError:
+ # The 'resources' keyword was only introduced after Zope 5.9.
+ context.registerClass(
+ meta_type=self.meta_type,
+ constructors=(manage_addToolForm, manage_addTool, self),
+ icon=self.icon)
if self.icon:
icon = os_path.split(self.icon)[1]
@@ -680,15 +687,27 @@ def __init__(self, meta_type, content_types, permission=None,
def initialize(self, context):
# Add only one meta type to the folder add list.
- context.registerClass(
- meta_type=self.meta_type,
- # This is a little sneaky: we add self to the
- # FactoryDispatcher under the name "contentinit".
- # manage_addContentType() can then grab it.
- constructors=(manage_addContentForm, manage_addContent,
- self) + self.extra_constructors,
- permission=self.permission,
- visibility=self.visibility)
+ # We add self to the FactoryDispatcher under the name 'contentinit'.
+ # manage_addContentType() can then grab it.
+ try:
+ context.registerClass(
+ meta_type=self.meta_type,
+ constructors=(
+ manage_addContentForm,
+ manage_addContent,
+ ) + self.extra_constructors,
+ resources=(self, ),
+ permission=self.permission,
+ visibility=self.visibility)
+ except TypeError:
+ # The 'resources' keyword was only introduced after Zope 5.9.
+ context.registerClass(
+ meta_type=self.meta_type,
+ constructors=(
+ manage_addContentForm, manage_addContent, self,
+ ) + self.extra_constructors,
+ permission=self.permission,
+ visibility=self.visibility)
for ct in self.content_types:
ct.__factory_meta_type__ = self.meta_type
From 3c6b815bbb2a9300984a7b50cb5ec5375bf4588e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Tue, 2 Apr 2024 21:54:07 +0900
Subject: [PATCH] Revive TRIGGER_WORKFLOW_METHOD support, ERP5 uses it
---
src/Products/DCWorkflow/DCWorkflow.py | 47 +++++++++++++++++++
src/Products/DCWorkflow/Transitions.py | 1 +
.../dtml/transition_properties.dtml | 10 ++++
src/Products/DCWorkflow/dtml/transitions.dtml | 3 +-
src/Products/DCWorkflow/exportimport.py | 2 +-
5 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/src/Products/DCWorkflow/DCWorkflow.py b/src/Products/DCWorkflow/DCWorkflow.py
index 9adf05c..d0306dc 100644
--- a/src/Products/DCWorkflow/DCWorkflow.py
+++ b/src/Products/DCWorkflow/DCWorkflow.py
@@ -38,6 +38,7 @@ from .Expression import createExprContext
from .interfaces import IDCWorkflowDefinition
from .Transitions import TRIGGER_AUTOMATIC
from .Transitions import TRIGGER_USER_ACTION
+from .Transitions import TRIGGER_WORKFLOW_METHOD
from .utils import Message as _
from .utils import modifyRolesForGroup
from .utils import modifyRolesForPermission
@@ -278,6 +279,52 @@ class DCWorkflowDefinition(WorkflowUIMixin, Folder):
raise Unauthorized(action)
self._changeStateOf(ob, tdef, kw)
+ @security.private
+ def isWorkflowMethodSupported(self, ob, method_id):
+ '''
+ Returns a true value if the given workflow method
+ is supported in the current state.
+ '''
+ sdef = self._getWorkflowStateOf(ob)
+ if sdef is None:
+ return 0
+ if method_id in sdef.transitions:
+ tdef = self.transitions.get(method_id, None)
+ if (tdef is not None and
+ tdef.trigger_type == TRIGGER_WORKFLOW_METHOD and
+ self._checkTransitionGuard(tdef, ob)):
+ return 1
+ return 0
+
+ @security.private
+ def wrapWorkflowMethod(self, ob, method_id, func, args, kw):
+ '''
+ Allows the user to request a workflow action. This method
+ must perform its own security checks.
+ '''
+ sdef = self._getWorkflowStateOf(ob)
+ if sdef is None:
+ raise WorkflowException('Object is in an undefined state')
+ if method_id not in sdef.transitions:
+ raise Unauthorized(method_id)
+ tdef = self.transitions.get(method_id, None)
+ if tdef is None or tdef.trigger_type != TRIGGER_WORKFLOW_METHOD:
+ raise WorkflowException(
+ 'Transition %s is not triggered by a workflow method'
+ % method_id)
+ if not self._checkTransitionGuard(tdef, ob):
+ raise Unauthorized(method_id)
+ res = func(*args, **kw)
+ try:
+ self._changeStateOf(ob, tdef)
+ except ObjectDeleted:
+ # Re-raise with a different result.
+ raise ObjectDeleted(res)
+ except ObjectMoved as ex:
+ # Re-raise with a different result.
+ raise ObjectMoved(ex.getNewObject(), res)
+ return res
+
@security.private
def isInfoSupported(self, ob, name):
'''
diff --git a/src/Products/DCWorkflow/Transitions.py b/src/Products/DCWorkflow/Transitions.py
index a6e1e6f..b4e012c 100644
--- a/src/Products/DCWorkflow/Transitions.py
+++ b/src/Products/DCWorkflow/Transitions.py
@@ -31,6 +31,7 @@ from .utils import _dtmldir
TRIGGER_AUTOMATIC = 0
TRIGGER_USER_ACTION = 1
+TRIGGER_WORKFLOW_METHOD = 2
class TransitionDefinition(SimpleItem):
diff --git a/src/Products/DCWorkflow/dtml/transition_properties.dtml b/src/Products/DCWorkflow/dtml/transition_properties.dtml
index d6b8a74..6a0803e 100644
--- a/src/Products/DCWorkflow/dtml/transition_properties.dtml
+++ b/src/Products/DCWorkflow/dtml/transition_properties.dtml
@@ -55,6 +55,16 @@ Initiated by user action
</td>
</tr>
+<tr>
+<th></th>
+<td>
+<dtml-let checked="trigger_type==2 and 'checked' or ' '">
+<input type="radio" name="trigger_type" value="2" &dtml-checked; />
+Initiated by WorkflowMethod
+</dtml-let>
+</td>
+</tr>
+
<tr>
<th align="left">Script (before)</th>
<td>
diff --git a/src/Products/DCWorkflow/dtml/transitions.dtml b/src/Products/DCWorkflow/dtml/transitions.dtml
index 4cdd3d3..37e949c 100644
--- a/src/Products/DCWorkflow/dtml/transitions.dtml
+++ b/src/Products/DCWorkflow/dtml/transitions.dtml
@@ -17,7 +17,8 @@
<td>
Destination state: <code><dtml-if new_state_id>&dtml-new_state_id;<dtml-else>(Remain in state)</dtml-if></code> <br />
Trigger: <dtml-var expr="(trigger_type == 0 and 'Automatic') or
- (trigger_type == 1 and 'User action')">
+ (trigger_type == 1 and 'User action') or
+ (trigger_type == 2 and 'WorkflowMethod')">
<br />
<dtml-if script_name>
Script (before): &dtml-script_name;
diff --git a/src/Products/DCWorkflow/exportimport.py b/src/Products/DCWorkflow/exportimport.py
index f17264d..2374b6e 100644
--- a/src/Products/DCWorkflow/exportimport.py
+++ b/src/Products/DCWorkflow/exportimport.py
@@ -37,7 +37,7 @@ from .interfaces import IDCWorkflowDefinition
from .utils import _xmldir
-TRIGGER_TYPES = ('AUTOMATIC', 'USER')
+TRIGGER_TYPES = ('AUTOMATIC', 'USER', 'METHOD' )
_FILENAME = 'workflows.xml'
--
2.42.0
From a037f2a2e2090dcd63b83af9b06427dd8c7e9536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Wed, 22 May 2024 23:58:45 +0900
Subject: [PATCH] Show Python Script source code in tracebacks
Expose a __loader__ in globals so that linecache module is able to use
it to display the source code.
This requires changing the "filename" used when compiling function,
because linecache uses code.co_filename as a cache key, so it's
necessary that each python script use a different filename.
WIP from https://github.com/zopefoundation/Products.PythonScripts/pull/65
---
CHANGES.rst | 2 +
src/Products/PythonScripts/PythonScript.py | 19 ++++++-
.../PythonScripts/tests/testPythonScript.py | 50 ++++++++++++++++++-
3 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/src/Products/PythonScripts/PythonScript.py b/src/Products/PythonScripts/PythonScript.py
index fe4223a..5cb7f37 100644
--- a/src/Products/PythonScripts/PythonScript.py
+++ b/src/Products/PythonScripts/PythonScript.py
@@ -16,7 +16,9 @@
Python code.
"""
+import importlib.abc
import importlib.util
+import linecache
import marshal
import os
import re
@@ -56,7 +58,7 @@
Python_magic = importlib.util.MAGIC_NUMBER
# This should only be incremented to force recompilation.
-Script_magic = 4
+Script_magic = 5
_log_complaint = (
'Some of your Scripts have stale code cached. Since Zope cannot'
' use this code, startup will be slightly slower until these Scripts'
@@ -97,6 +99,16 @@ def manage_addPythonScript(self, id, title='', file=None, REQUEST=None,
return ''
+class PythonScriptLoader(importlib.abc.Loader):
+ """PEP302 loader to display source code in tracebacks
+ """
+ def __init__(self, source):
+ self._source = source
+
+ def get_source(self, name):
+ return self._source
+
+
class PythonScript(Script, Historical, Cacheable):
"""Web-callable scripts written in a safe subset of Python.
@@ -234,7 +246,7 @@ def _compile(self):
self._params,
body=self._body or 'pass',
name=self.id,
- filename=self.meta_type,
+ filename=getattr(self, '_filepath', None) or self.get_filepath(),
globalize=bind_names)
code = compile_result.code
@@ -261,6 +273,7 @@ def _compile(self):
fc.co_argcount)
self.Python_magic = Python_magic
self.Script_magic = Script_magic
+ linecache.clearcache()
self._v_change = 0
def _newfun(self, code):
@@ -331,6 +344,8 @@ def _exec(self, bound_names, args, kw):
PythonScriptTracebackSupplement, self, -1)
safe_globals['__file__'] = getattr(
self, '_filepath', None) or self.get_filepath()
+ safe_globals['__loader__'] = PythonScriptLoader(self._body)
+
function = types.FunctionType(
function_code, safe_globals, None, function_argument_definitions)
diff --git a/src/Products/PythonScripts/tests/testPythonScript.py b/src/Products/PythonScripts/tests/testPythonScript.py
index 60ef6c3..7cd2266 100644
--- a/src/Products/PythonScripts/tests/testPythonScript.py
+++ b/src/Products/PythonScripts/tests/testPythonScript.py
@@ -15,6 +15,7 @@
import io
import os
import sys
+import traceback
import unittest
import warnings
from urllib.error import HTTPError
@@ -241,7 +242,8 @@ def test_manage_DAVget(self):
self.assertEqual(ps.read(), ps.manage_DAVget())
def test_PUT_native_string(self):
- ps = makerequest(self._filePS('complete'))
+ container = DummyFolder('container')
+ ps = makerequest(self._filePS('complete').__of__(container))
self.assertEqual(ps.title, 'This is a title')
self.assertEqual(ps.body(), 'print(foo+bar+baz)\nreturn printed\n')
self.assertEqual(ps.params(), 'foo, bar, baz=1')
@@ -265,7 +267,8 @@ def test_PUT_native_string(self):
self.assertEqual(ps.params(), 'oops')
def test_PUT_bytes(self):
- ps = makerequest(self._filePS('complete'))
+ container = DummyFolder('container')
+ ps = makerequest(self._filePS('complete').__of__(container))
self.assertEqual(ps.title, 'This is a title')
self.assertEqual(ps.body(), 'print(foo+bar+baz)\nreturn printed\n')
self.assertEqual(ps.params(), 'foo, bar, baz=1')
@@ -588,3 +591,46 @@ def test_PythonScript_proxyroles_nonmanager(self):
# Cleanup
noSecurityManager()
+
+
+class TestTraceback(FunctionalTestCase, PythonScriptTestBase):
+
+ def _format_exception(self):
+ return "".join(traceback.format_exception(*sys.exc_info()))
+
+ def test_source_code_in_traceback(self):
+ ps = self._newPS("1 / 0")
+ try:
+ ps()
+ except ZeroDivisionError:
+ formatted_exception = self._format_exception()
+ self.assertIn("1 / 0", formatted_exception)
+
+ ps.write("2 / 0")
+ try:
+ ps()
+ except ZeroDivisionError:
+ formatted_exception = self._format_exception()
+ self.assertIn("2 / 0", formatted_exception)
+
+ def test_multiple_scripts_in_traceback(self):
+ from Products.PythonScripts.PythonScript import manage_addPythonScript
+
+ script1_body = "container.script2()"
+ manage_addPythonScript(
+ self.folder,
+ "script1",
+ file=script1_body,
+ )
+ script2_body = "1 / 0"
+ manage_addPythonScript(
+ self.folder,
+ "script2",
+ file=script2_body,
+ )
+ try:
+ self.folder.script1()
+ except ZeroDivisionError:
+ formatted_exception = self._format_exception()
+ self.assertIn(script1_body, formatted_exception)
+ self.assertIn(script2_body, formatted_exception)
From 21a91db138cca3ada0e4dff475b061066362410c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Sat, 17 Feb 2024 23:25:43 +0900
Subject: [PATCH] backport changes from 0.52.29
We can not use 0.52.29 directly because it does not have a setup.py
and our buildout / setuptools tooling is too old.
---
src/SOAPpy/Client.py | 3 ++-
src/SOAPpy/Types.py | 2 ++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/SOAPpy/Client.py b/src/SOAPpy/Client.py
index e86c5ec..d2bbefb 100644
--- a/src/SOAPpy/Client.py
+++ b/src/SOAPpy/Client.py
@@ -45,6 +45,7 @@
ident = '$Id: Client.py 1496 2010-03-04 23:46:17Z pooryorick $'
from .version import __version__
+from io import StringIO
#import xml.sax
import urllib.request, urllib.parse, urllib.error
@@ -152,7 +153,7 @@ class HTTP:
return -1, e.line, None
self.headers = response.msg
- self.file = response.fp
+ self.file = StringIO(response.fp.read().decode('utf-8'))
return response.status, response.reason, response.msg
def close(self):
diff --git a/src/SOAPpy/Types.py b/src/SOAPpy/Types.py
index de9dcac..cf08d17 100644
--- a/src/SOAPpy/Types.py
+++ b/src/SOAPpy/Types.py
@@ -1451,6 +1451,8 @@ class arrayType(collections.UserList, compoundType):
def __getitem__(self, item):
try:
return self.data[int(item)]
+ except TypeError:
+ return self.data[item]
except ValueError:
return getattr(self, item)
--
2.42.0
From c56146829ab065183c709229a9daa682cc445212 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Fri, 26 Apr 2024 15:09:39 +0900
Subject: [PATCH] fix loading font for ean13
use same technique as for code128
---
hubarcode/ean13/renderer.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/hubarcode/ean13/renderer.py b/hubarcode/ean13/renderer.py
index 654501e..ff5f518 100644
--- a/hubarcode/ean13/renderer.py
+++ b/hubarcode/ean13/renderer.py
@@ -78,8 +78,10 @@ class EAN13Renderer:
# Draw the text
font_size = font_sizes.get(bar_width, 24)
- # Use relative name, PIL will do searching for us
- fontfile = os.path.join("fonts", "courR%02d.pil" % font_size)
+ # Locate and load the font file relative to the module
+ ean13dir, _ = os.path.split(__file__)
+ rootdir, _ = os.path.split(ean13dir)
+ fontfile = os.path.join(rootdir, "fonts", "courR%02d.pil" % font_size)
font = ImageFont.load_path(fontfile)
draw = ImageDraw.Draw(img)
--
2.42.0
This diff is collapsed.
diff -ur urlnorm-1.1.4.orig/setup.py urlnorm-1.1.4/setup.py
--- urlnorm-1.1.4.orig/setup.py 2016-08-05 20:07:24.000000000 +0200
+++ urlnorm-1.1.4/setup.py 2022-10-21 09:32:35.377477901 +0200
@@ -9,8 +9,15 @@
description="Normalize a URL to a standard unicode encoding",
py_modules=['urlnorm'],
license='MIT License',
+ install_requires=['six'],
author='Jehiah Czebotar',
author_email='jehiah@gmail.com',
url='http://github.com/jehiah/urlnorm',
download_url="http://github.com/downloads/jehiah/urlnorm/urlnorm-%s.tar.gz" % version,
+ classifiers=[
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.6',
+ ],
)
diff -ur urlnorm-1.1.4.orig/urlnorm.py urlnorm-1.1.4/urlnorm.py
--- urlnorm-1.1.4.orig/urlnorm.py 2016-08-05 20:06:32.000000000 +0200
+++ urlnorm-1.1.4/urlnorm.py 2022-10-21 09:32:35.377477901 +0200
@@ -41,6 +41,10 @@
- more fine-grained authority parsing and normalisation
"""
+from __future__ import absolute_import
+from six import unichr
+import six
+from six.moves import range
__license__ = """
Copyright (c) 1999-2002 Mark Nottingham <mnot@pobox.com>
Copyright (c) 2010 Jehiah Czebotar <jehiah@gmail.com>
@@ -67,8 +71,7 @@
# also update in setup.py
__version__ = "1.1.4"
-from urlparse import urlparse, urlunparse
-from string import lower
+from six.moves.urllib.parse import urlparse, urlunparse, unquote
import re
@@ -108,8 +111,8 @@
qs_unsafe_list = set('?&=+%#')
fragment_unsafe_list = set('+%#')
path_unsafe_list = set('/?;%+#')
-_hextochr = dict(('%02x' % i, chr(i)) for i in range(256))
-_hextochr.update(('%02X' % i, chr(i)) for i in range(256))
+_hextochr = dict((b'%02x' % i, six.int2byte(i)) for i in range(256))
+_hextochr.update((b'%02X' % i, six.int2byte(i)) for i in range(256))
def unquote_path(s):
@@ -132,22 +135,23 @@
"""unquote percent escaped string except for percent escape sequences that are in unsafe_list"""
# note: this build utf8 raw strings ,then does a .decode('utf8') at the end.
# as a result it's doing .encode('utf8') on each block of the string as it's processed.
- res = _utf8(s).split('%')
- for i in xrange(1, len(res)):
+ unsafe_list = [_utf8(i) for i in unsafe_list]
+ res = _utf8(s).split(b'%')
+ for i in range(1, len(res)):
item = res[i]
try:
raw_chr = _hextochr[item[:2]]
if raw_chr in unsafe_list or ord(raw_chr) < 20:
# leave it unescaped (but uppercase the percent escape)
- res[i] = '%' + item[:2].upper() + item[2:]
+ res[i] = b'%' + item[:2].upper() + item[2:]
else:
res[i] = raw_chr + item[2:]
except KeyError:
- res[i] = '%' + item
+ res[i] = b'%' + item
except UnicodeDecodeError:
# note: i'm not sure what this does
res[i] = unichr(int(item[:2], 16)) + item[2:]
- o = "".join(res)
+ o = b"".join(res)
return _unicode(o)
@@ -160,7 +164,7 @@
def norm_tuple(scheme, authority, path, parameters, query, fragment):
"""given individual url components, return its normalized form"""
- scheme = lower(scheme)
+ scheme = scheme.lower()
if not scheme:
raise InvalidUrl('missing URL scheme')
authority = norm_netloc(scheme, authority)
@@ -203,7 +207,7 @@
return '/'
return path
-MAX_IP = 0xffffffffL
+MAX_IP = 0xffffffff
def int2ip(ipnum):
@@ -238,7 +242,7 @@
if '.' not in host and not (host[0] == '[' and host[-1] == ']'):
raise InvalidUrl('host %r is not valid' % host)
- authority = lower(host)
+ authority = host.lower()
if 'xn--' in authority:
subdomains = [_idn(subdomain) for subdomain in authority.split('.')]
authority = '.'.join(subdomains)
@@ -260,14 +264,14 @@
def _utf8(value):
- if isinstance(value, unicode):
+ if isinstance(value, six.text_type):
return value.encode("utf-8")
assert isinstance(value, str)
return value
def _unicode(value):
- if isinstance(value, str):
+ if isinstance(value, six.binary_type):
return value.decode("utf-8")
- assert isinstance(value, unicode)
+ assert isinstance(value, six.text_type)
return value
# ethtool - query or control network driver and hardware settings
# https://www.kernel.org/pub/software/network/ethtool
[buildout]
extends =
../xz-utils/buildout.cfg
parts = ethtool
[ethtool]
recipe = slapos.recipe.cmmi
url = https://www.kernel.org/pub/software/network/ethtool/ethtool-4.11.tar.xz
md5sum = 16d38f4ebe23e44f96f7d8b38ed3652c
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
......@@ -8,6 +8,7 @@ extends =
../perl/buildout.cfg
../tar/buildout.cfg
../binutils/buildout.cfg
../xz-utils/buildout.cfg
parts =
gcc-10.5
......@@ -56,7 +57,7 @@ post-install =
chmod +x ld
environment =
LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:${xz-utils:location}/bin:%(PATH)s
[gcc-common:os.path.isdir('/usr/lib/arm-linux-gnueabihf') and os.access('/proc/device-tree/model', os.R_OK) and 'OLinuXino' in open('/proc/device-tree/model').read()]
extra-configure-options =
......
......@@ -18,14 +18,13 @@ shared = true
url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs10031/ghostscript-10.03.1.tar.gz
md5sum = 01c515f190de2d8dcce4767407720855
pkg_config_depends = ${libidn:location}/lib/pkgconfig:${libtiff:location}/lib/pkgconfig:${libjpeg:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}
# XXX --with-tessdata work arounds a slaprunner bug of having softwares installed in a path containing //
configure-options =
--disable-cups
--with-system-libtiff
--without-so
--without-x
--with-drivers=FILES
--with-tessdata=$(python -c 'print("""${:tessdata-location}""".replace("//", "/"))')
--with-tessdata=${:tessdata-location}
environment =
PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
......
......@@ -33,6 +33,9 @@ environment =
PATH=${curl:location}/bin:${gettext:location}/bin:${tar:location}/bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${curl:location}/lib -Wl,-rpath=${libexpat:location}/lib
patch-options = -p1
patches =
${:_profile_base_location_}/prevent-git-fetch-pack-segfault.patch#f18e84a5ff8951aad094eddf32fbf8a1
[gitweb]
<= git
......
From 93be30997053105b5e3bad9203e5d07cbea0f0e6 Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <alain.takoudjou@nexedi.com>
Date: Fri, 7 Jun 2024 12:13:50 +0200
Subject: [PATCH] prevent-git-fetch-pack-segfault
---
fetch-pack.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fetch-pack.c b/fetch-pack.c
index 65c1ff4bb4..a9b1558ce2 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -1037,7 +1037,7 @@ static int get_pack(struct fetch_pack_args *args,
cmd.git_cmd = 1;
if (start_command(&cmd))
die(_("fetch-pack: unable to fork off %s"), cmd_name);
- if (do_keep && (pack_lockfiles || fsck_objects)) {
+ if (do_keep && (pack_lockfiles /*|| fsck_objects*/)) {
int is_well_formed;
char *pack_lockfile = index_pack_lockfile(cmd.out, &is_well_formed);
--
2.34.1
......@@ -96,6 +96,15 @@ patches +=
https://lab.nexedi.com/kirr/go/commit/6dfc2256e2cd1ad46f3a5e9da85af6a70be7fba3.patch#fa9de83838b198490abec0a6a924ad8d
https://lab.nexedi.com/kirr/go/commit/28fbdd01d6c89db1e81e12bd05910fdec4c3b9b3.patch#c75e2eb7aaaeffdd3c507ec1beacd3ca
[golang1.15]
<= golang-common-pre-1.19
url = https://go.dev/dl/go1.15.15.src.tar.gz
md5sum = 05fedd8289291eb2d91cd0c092b41aaa
# go1.1 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.16]
<= golang-common-pre-1.19
url = https://golang.org/dl/go1.16.15.src.tar.gz
......
......@@ -10,11 +10,12 @@ parts =
LDFLAGS = -L${hdf5:location}/lib -Wl,-rpath=${hdf5:location}/lib
CPPFLAGS = -I${hdf5:location}/include
LD_LIBRARY_PATH=${hdf5:location}/lib
HDF5_DIR=${hdf5:location}
[h5py]
recipe = zc.recipe.egg:custom
egg = h5py
setup-eggs =
setup-eggs =
${cython:egg}
${numpy:egg}
pkgconfig
......
......@@ -7,10 +7,14 @@ parts =
[hdf5]
recipe = slapos.recipe.cmmi
shared = true
url = https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0-patch1/src/hdf5-1.10.0-patch1.tar.bz2
md5sum = f6d980febe2c35c11670a9b34fa3b487
url = https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.3/src/hdf5-1.14.3.tar.gz
md5sum = 075d923171c7785b2ee7d820b0ef8707
configure-options =
--with-zlib=${zlib:location}
environment =
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
[hdf5:python2]
url = https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0-patch1/src/hdf5-1.10.0-patch1.tar.bz2
md5sum = f6d980febe2c35c11670a9b34fa3b487
......@@ -11,7 +11,7 @@ parts =
<= numpy-env
[ipython]
recipe = zc.recipe.egg:custom
recipe = zc.recipe.egg
egg = ipython
environment = ipython-env
setup-eggs =
......
......@@ -75,7 +75,7 @@ class ERP5Kernel(Kernel):
self.title = None
# Allowed HTTP request code list for making request to erp5 from Kernel
# This list should be to used check status_code before making requests to erp5
self.allowed_HTTP_request_code_list = range(500, 511)
self.allowed_HTTP_request_code_list = list(range(500, 511))
# Append request code 200 in the allowed HTTP status code list
self.allowed_HTTP_request_code_list.append(200)
......
......@@ -14,9 +14,8 @@ parts +=
# Always build GCC for Fortran (see openblas).
max_version = 0
[jupyter]
[jupyter:python2]
extra-eggs =
python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
recipe = slapos.recipe.build:download
......@@ -46,7 +45,7 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key openssl_output openssl-output:openssl
key python_executable jupyter:python_executable
key python_executable jupyter:python-executable
key jupyter_config_location jupyter-notebook-config:location
key jupyter_config_filename jupyter-notebook-config:filename
key jupyter_set_password_location jupyter-set-password:location
......@@ -59,7 +58,7 @@ context =
key custom_js_filename custom-js:filename
key monitor_template_rendered buildout:directory
[versions]
[versions:python2]
Pygments = 2.2.0
ipykernel = 4.5.2
ipython = 5.3.0
......
......@@ -15,11 +15,11 @@
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = fd7ed44da8d8723983b8666df2971a36
md5sum = c335782940a8f3b1ff7d4280aeec336e
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 9d579353b579b6e488ae6330c7f4ad68
md5sum = 10b9a9892d50c5d085ff0be5936ab88b
[jupyter-set-password]
filename = jupyter_set_password.cgi.jinja
......@@ -27,7 +27,7 @@ md5sum = ac10fbcf790bd8e58750cfdd069812d2
[erp5-kernel]
filename = ERP5kernel.py
md5sum = 7d5309fe79afbcb455c0d8181b42e56c
md5sum = da04b99b70b2e327c9e9b4cdd056098e
[kernel-json]
filename = kernel.json.jinja
......
......@@ -55,7 +55,7 @@ key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/jupyter-lab
{{ bin_directory }}/jupyter-notebook
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
......@@ -69,15 +69,19 @@ environment =
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
JUPYTERLAB_DIR=${directory:jupyterlab-dir}
LANG=C.UTF-8
[jupyter-notebook-config]
recipe = slapos.recipe.template:jinja2
url = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
output = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
output = ${directory:jupyter_config_dir}/jupyter_server_config.py
context =
raw config_cfg ${buildout:directory}/knowledge0.cfg
[jupyter-notebook-config:python2]
output = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
......@@ -95,12 +99,13 @@ jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
jupyterlab-dir = ${:jupyter_dir}/lab
[jupyter_notebook]
# This part is called like this because knowledge0.write uses the part name for
# the section name in the config file.
recipe = slapos.cookbook:zero-knowledge.write
password =
password =
filename = knowledge0.cfg
[read-knowledge0]
......@@ -136,7 +141,7 @@ output = ${directory:erp5_kernel_dir}/kernel.json
context =
raw python_executable {{ python_executable }}
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
......
'''
This script initializes Jupyter's configuration such as passwords and other
things. It is run by IPython hence why it can use functions like get_config().
'''
import os
import ssl
import sys
import six
from six.moves.configparser import ConfigParser
if six.PY3:
from jupyter_server.auth import passwd
import secrets
random_password = secrets.token_hex
else:
from notebook.auth import passwd
import random
def random_password(length=10):
result = ""
for i in range(0, length):
result = result + chr(random.randint(0, 25) + ord('a'))
return result
knowledge_0 = '{{ config_cfg }}'
if not os.path.exists(knowledge_0):
print ("Your software does <b>not</b> embed 0-knowledge.\n"
"This interface is useless in this case</body></html>")
exit(0)
c = get_config()
parser = ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password") or \
parser.get("jupyter_notebook", "password") == "":
parser.set("jupyter_notebook", "password", random_password())
if six.PY3: # This supports old jupyterlab on python2 and recent jupyterlab on python3
import pathlib
import jupyterlab
jupyterlab_dir = pathlib.Path(os.environ['JUPYTERLAB_DIR'])
# symlink all schemas in a folder, jupyter seems to assume that everything is installed
# in the same place.
schemas_dir = jupyterlab_dir / 'schemas'
if not schemas_dir.exists():
schemas_dir.mkdir()
for p in sys.path:
for schema in (pathlib.Path(p) / 'share' / 'jupyter' / 'lab' / 'schemas').glob('*/'):
dest = (schemas_dir / schema.name)
if dest.exists():
dest.unlink()
dest.symlink_to(schema)
c.LabServerApp.schemas_dir = str(schemas_dir)
# static really needs to be a sub-folder of $JUPYTERLAB_DIR
static = pathlib.Path(jupyterlab.__file__).parent.parent / 'share' / 'jupyter' / 'lab' / 'static'
static_dir = jupyterlab_dir / 'static'
if static_dir.exists():
static_dir.unlink()
static_dir.symlink_to(static)
c.LabServerApp.themes_dir = str(pathlib.Path(jupyterlab.__file__).parent / 'themes')
c.ServerApp.jpserver_extensions = {
'notebook': True,
'jupyter_lsp':True,
'jupyter_server_terminals': True,
'jupyterlab': True,
'notebook_shim': True,
}
c.ServerApp.password = passwd(parser.get("jupyter_notebook", "password"))
else:
c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
c.NotebookApp.ssl_options = {
'ssl_version': ssl.PROTOCOL_TLSv1_2,
}
with open(knowledge_0, 'w') as file:
parser.write(file)
'''
This script initializes Jupyter's configuration such as passwords and other
things. It is run by IPython hence why it can use functions like get_config().
'''
import ConfigParser
import random
from notebook.auth import passwd
import os
import ssl
def random_password(length = 10):
result = ""
for i in range(0, length):
result = result + chr(random.randint(0, 25) + ord('a'))
return result
knowledge_0 = '{{ config_cfg }}'
if not os.path.exists(knowledge_0):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
c = get_config()
parser = ConfigParser.ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password") or \
parser.get("jupyter_notebook", "password") == "":
parser.set("jupyter_notebook", "password", random_password())
c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
c.NotebookApp.ssl_options = {
'ssl_version': ssl.PROTOCOL_TLSv1_2,
}
with open(knowledge_0, 'w') as file:
parser.write(file)
......@@ -3,7 +3,7 @@ extends =
../numpy/openblas.cfg
../matplotlib/buildout.cfg
../ipython/buildout.cfg
../python-cffi/buildout.cfg
../python-argon2-cffi/buildout.cfg
../python-pyzmq/buildout.cfg
../scipy/buildout.cfg
../scikit-learn/buildout.cfg
......@@ -15,10 +15,6 @@ parts =
jupyter
jupyter-notebook-scripts
[argon2-cffi]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = ${python-cffi:egg}
[jupyter-env]
<= numpy-env
......@@ -74,6 +70,7 @@ scripts =
jupyter-migrate
jupyter-troubleshoot
jupyter-run
python-executable = ${buildout:bin-directory}/${:interpreter}
[jupyter-notebook-initialized-scripts]
recipe = zc.recipe.egg:scripts
......@@ -82,9 +79,7 @@ environment = jupyter-env
scripts =
jupyter-nbconvert
jupyter-nbextension
jupyter-notebook
jupyter-serverextension
jupyter-lab
jupyter-labextension
jupyter-labhub
......
From 386b9a90dec9be74a271258833acb8c7ae993eed Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <alain.takoudjou@nexedi.com>
Date: Tue, 23 Apr 2024 20:51:04 +0200
Subject: [PATCH] prefer use python 3 for tests
---
tests/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6f8a18ec0..04ccf99e7 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,3 +1,4 @@
+set(Python_ADDITIONAL_VERSIONS 3 2.7)
FIND_PACKAGE(PythonInterp)
IF(NOT PYTHONINTERP_FOUND)
--
2.42.0
[buildout]
extends =
../defaults.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../pkgconfig/buildout.cfg
../pcre/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts = libgit2
[libgit2]
recipe = slapos.recipe.cmmi
url = https://github.com/libgit2/libgit2/archive/refs/tags/v${:version}.tar.gz
md5sum = 9f4ca15249e703ab88cbc929187750cd
version = 1.1.0
shared = true
configure-command =
cmake
configure-options =
-Bbuild
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DCMAKE_C_FLAGS="-I${openssl:location}/include -I${zlib:location}/include -I${pcre:location}/include"
-DCMAKE_INSTALL_RPATH=${zlib:location}/lib:${openssl:location}/lib:${pcre:location}/lib
-G"Unix Makefiles"
patch-options = -p1
patches =
${:_profile_base_location_}/0001-prefer-use-python-3-for-tests.patch#6f2a6e83db45b33fc7da86279f06595b
make-options = -C build
environment =
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
PATH=${python3:location}/bin:${pkgconfig:location}/bin:${cmake:location}/bin:%(PATH)s
LDFLAGS=-L${openssl-1.0:location}/lib -Wl,-rpath=${openssl:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib
[buildout]
extends =
../gnutls/buildout.cfg
../gnupg/buildout.cfg
parts =
libgpgme
[libgpgme]
recipe = slapos.recipe.cmmi
url = https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-1.23.2.tar.bz2
md5sum = 01a8c05b409847e87daf0543e91f8c37
configure-options =
--disable-gpg-test
--with-libgpg-error-prefix=${libgpg-error:location}
--with-libassuan-prefix=${libassuan:location}
make-options =
PYTHONS=
environment =
LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib -Wl,-rpath=${libassuan:location}/lib
......@@ -3,6 +3,7 @@
extends =
../bzip2/buildout.cfg
../cmake/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts = libzip
......@@ -22,7 +23,7 @@ configure-options =
-DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
make-options = -C build
environment =
PATH=${cmake:location}/bin:%(PATH)s
PATH=${cmake:location}/bin:${xz-utils:location}/bin:%(PATH)s
CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
LDFLAGS=-Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
......@@ -40,6 +40,7 @@ need-matplotlibrc = ${matplotlibrc:location}
[versions]
matplotlib = 2.1.2
cycler = 0.11.0
matplotlib-inline = 0.1.6:whl
[versions:sys.version_info < (3,8)]
cycler = 0.10.0
......@@ -6,6 +6,7 @@ extends =
../file/buildout.cfg
../zlib/buildout.cfg
../ncurses/buildout.cfg
../xz-utils/buildout.cfg
[nano]
recipe = slapos.recipe.cmmi
......@@ -15,6 +16,7 @@ md5sum = 12784a5c245518d7580125ebbd6b7601
# The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
# are installed on the system.
environment=
PATH=${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG=false
NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
CPPFLAGS=-I${file:location}/include -I${zlib:location}/include
......
......@@ -21,7 +21,7 @@ environment = numpy-env
eggs = ${cython:egg}
[versions]
numpy = 1.22.0
numpy = 1.24.4
[numpy:sys.version_info < (3,8)]
depends =
......
# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
[buildout]
extends =
../defaults.cfg
test.cfg
[python]
part = python2.7
[openssl]
<= openssl-1.1
../../stack/slapos-py2.cfg
......@@ -5,9 +5,10 @@ parts =
open62541
extends =
../cmake/buildout.cfg
../git/buildout.cfg
../mbedtls/buildout.cfg
../patch/buildout.cfg
../python3/buildout.cfg
../gcc/buildout.cfg
../defaults.cfg
[gcc]
......@@ -27,6 +28,7 @@ pre-configure =
# remove old FindPython3 which does not know about python > 3.8 to
# use FindPython3 bundled with slapos' cmake
rm tools/cmake/FindPython3.cmake tools/cmake/FindPython/Support.cmake
${git:location}/bin/git clone -b v1.1.6 https://github.com/LiamBindle/MQTT-C.git deps/mqtt-c
configure-command =
${cmake:location}/bin/cmake
configure-options =
......@@ -34,9 +36,20 @@ configure-options =
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DUA_ENABLE_PUBSUB=ON
-DUA_ENABLE_PUBSUB_MONITORING=ON
-DUA_ENABLE_PUBSUB_ETH_UADP=ON
-DUA_ENABLE_SUBSCRIPTIONS=ON
-DUA_NAMESPACE_ZERO=REDUCED
-DUA_ENABLE_ENCRYPTION=MBEDTLS
-DUA_ENABLE_ENCRYPTION_MBEDTLS=ON
-DMBEDTLS_INCLUDE_DIRS=${mbedtls:location}/include
-DMBEDTLS_LIBRARY=${mbedtls:location}/lib/libmbedtls.so
-DMBEDX509_LIBRARY=${mbedtls:location}/lib/libmbedx509.so
-DMBEDCRYPTO_LIBRARY=${mbedtls:location}/lib/libmbedcrypto.so
-DUA_ENABLE_PUBSUB_INFORMATIONMODEL=ON
-DUA_ENABLE_PUBSUB_MQTT=ON
post-install =
cp src/pubsub/*.h deps/open62541_queue.h @@LOCATION@@/include
environment =
PATH=${gcc:prefix}/bin:${python3:location}/bin:${patch:location}/bin:%(PATH)s
LDFLAGS=-L${mbedtls:location}/lib -Wl,-rpath=${mbedtls:location}/lib
[buildout]
extends =
../git/buildout.cfg
../mbedtls/buildout.cfg
../open62541/buildout.cfg
../defaults.cfg
parts = osie-coupler
[gcc]
# we want this SR to use a fixed gcc (so that we have the same gcc as open62541 component)
max_version = 0
[osie-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/osie.git
revision = dd9aea8
[osie-coupler]
recipe = slapos.recipe.cmmi
path = ${osie-repository:location}/coupler
bin_dir = ${:path}/bin/
environment =
PATH=${gcc:prefix}/bin:/usr/bin
C_INCLUDE_PATH=${open62541:location}/include:${open62541:location}/deps:${open62541:location}/src/pubsub
LDFLAGS=-L${open62541:location}/lib -Wl,-rpath=${open62541:location}/lib -L ${mbedtls:location}/lib -Wl,-rpath=${mbedtls:location}/lib
configure-command = true
......@@ -34,3 +34,8 @@ rpath =
${libjpeg:location}/lib
${libtiff:location}/lib
${zlib:location}/lib
Pillow-patches = ${:_profile_base_location_}/../../component/egg-patch/Pillow/0001-set-metadata-in-setup.py-for-compatibility-with-old-.patch#0a06cc5a94d3db24688938731e4b15e2
Pillow-patch-options = -p1
[pillow-python:python2]
Pillow-patches =
......@@ -33,7 +33,7 @@ configure-options =
# build core PostgreSQL + pg_trgm contrib extension for GitLab
# unaccent contrib extension is for peertube
# citext contrib extension is for metabase
make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install
make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install && make -C contrib/btree_gist/ install
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/include
......
# SlapOS software release to test pygolang/py2 on Nexedi testing infrastructure.
[buildout]
extends = test.cfg
[python]
part = python2.7
[openssl]
<= openssl-1.1
extends =
test.cfg
../../stack/slapos-py2.cfg
......@@ -55,4 +55,3 @@ ipython-genutils = 0.2.0
Pygments = 2.5.2
prompt-toolkit = 1.0.18
pickleshare = 0.7.5
ptyprocess = 0.6.0
......@@ -5,19 +5,27 @@ parts =
extends =
../patch/buildout.cfg
[astroid]
[pylint]
recipe = zc.recipe.egg
egg = pylint
[pylint:python2]
recipe = zc.recipe.egg:custom
egg = astroid
patches =
${:_profile_base_location_}/astroid-six_moves_import_error.patch#377beb0c50f52b9608bb6be7bf93096e
${:_profile_base_location_}/pylint-super_on_old_class.patch#cb0c3f8c091bf4980be395c917edc435
${:_profile_base_location_}/pylint-redefining-builtins-modules.patch#043defc6e9002ac48b40e078797d4d17
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
[pylint]
[astroid]
recipe = zc.recipe.egg
egg = astroid
[astroid:python2]
recipe = zc.recipe.egg:custom
egg = pylint
patches =
${:_profile_base_location_}/pylint-super_on_old_class.patch#cb0c3f8c091bf4980be395c917edc435
${:_profile_base_location_}/pylint-redefining-builtins-modules.patch#043defc6e9002ac48b40e078797d4d17
${:_profile_base_location_}/astroid-six_moves_import_error.patch#377beb0c50f52b9608bb6be7bf93096e
${:_profile_base_location_}/fix-import-six.moves.urllib.request-on-astroid-1.3.8.patch#266139a893d0eba377ac510fb0fa75f1
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
From 67abf302360eab857fb02d1e83a97aff86f31aa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Tue, 9 Apr 2024 11:04:38 +0900
Subject: [PATCH] fix "import six.moves.urllib.request" on astroid 1.3.8
---
astroid/brain/pysix_moves.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/astroid/brain/pysix_moves.py b/astroid/brain/pysix_moves.py
index 4a82b499..9bf31146 100644
--- a/astroid/brain/pysix_moves.py
+++ b/astroid/brain/pysix_moves.py
@@ -164,6 +164,7 @@ if sys.version_info[0] == 2:
urllib_parse = UrllibParse()
urllib_error = UrllibError()
+ urllib_request = UrllibRequest()
urllib = DummyModule()
urllib.request = UrllibRequest()
urllib.parse = UrllibParse()
--
2.42.0
[buildout]
extends =
../../component/cython/buildout.cfg
[PyStemmer]
recipe = zc.recipe.egg:custom
egg = PyStemmer
setup-eggs =
${cython:egg}
setuptools-dso
[buildout]
extends =
../python-cffi/buildout.cfg
parts = argon2-cffi
[argon2-cffi]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = ${python-cffi:egg}
[buildout]
parts =
python-ldap
python-ldap-python
extends =
../cyrus-sasl/buildout.cfg
../openldap/buildout.cfg
......@@ -10,19 +10,25 @@ extends =
[python-ldap-python]
recipe = zc.recipe.egg:custom
egg = python-ldap
patches =
${:_profile_base_location_}/python-ldap-no_default_dirs.patch#959115f13f1de5c63654c69b8dfacd69
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
rpath =
${openldap:location}/lib
${cyrus-sasl:location}/lib
${openssl:location}/lib
include-dirs =
${openldap:location}/include
${cyrus-sasl:location}/include/sasl
${cyrus-sasl:location}/include
${openssl:location}/include
library-dirs =
${openldap:location}/lib
${cyrus-sasl:location}/lib
${openssl:location}/lib
[python-ldap-python:python2]
patches =
${:_profile_base_location_}/python-ldap-no_default_dirs.patch#959115f13f1de5c63654c69b8dfacd69
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
include-dirs =
${openldap:location}/include
${cyrus-sasl:location}/include/sasl
${openssl:location}/include
......@@ -16,3 +16,5 @@ egg = pyzmq
environment = python-pyzmq-env
rpath =
${libzmq:location}/lib
setup-eggs =
packaging
......@@ -21,9 +21,19 @@ setup-eggs =
pkgconfig
pathlib2
setuptools-scm
toml
tomli
environment = python-xmlsec-env
[python-xmlsec:python2]
setup-eggs =
${lxml-python:egg}
pkgconfig
pathlib2
setuptools-scm
toml
[python-xmlsec-env]
PKG_CONFIG=${pkgconfig:location}/bin/pkg-config
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${xmlsec:location}/lib/pkgconfig
......
......@@ -19,6 +19,7 @@ extends =
../zlib/buildout.cfg
../glib/buildout.cfg
../ceph/buildout.cfg
../xz-utils/buildout.cfg
[gcc]
min_version = 7.4
......@@ -63,7 +64,7 @@ PKG_CONFIG_PATH-rbd = :${librbd:location}/lib/pkgconfig
environment =
CFLAGS=${:CFLAGS-rbd} -I${gettext:location}/include -I${libaio:location}/include -I${liburing:location}/include -I${libcap-ng:location}/include
LDFLAGS=${:LDFLAGS-rbd} -L${gettext:location}/lib -L${libaio:location}/lib -L${libcap-ng:location}/lib -Wl,-rpath=${libcap-ng:location}/lib -Wl,-rpath=${glib:location}/lib -Wl,-rpath=${gnutls:location}/lib -Wl,-rpath=${nettle:location}/lib -Wl,-rpath=${pixman:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${libpng:location}/lib -Wl,-rpath=${libaio:location}/lib -Wl,-rpath=${liburing:location}/lib -Wl,-rpath=${libcap-ng:location}/lib
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${glib:pkg_config_depends}:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig${:PKG_CONFIG_PATH-rbd}
[qemu:python2 or [int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]]
......
......@@ -8,20 +8,18 @@ extends =
../tcl/buildout.cfg
[redis]
<= redis28
[redis28]
recipe = slapos.recipe.cmmi
url = http://download.redis.io/releases/redis-2.8.24.tar.gz
md5sum = 7b6eb6e4ccc050c351df8ae83c55a035
url = https://download.redis.io/releases/redis-6.2.9.tar.gz
md5sum = 5251b9cfe7d799e6b63b7bd07410f97c
configure-command = true
prefix =
prefix =
make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
V=1
make-targets =
make-targets =
install
test
# There are many tests failing, so disable make test for now
# test
environment =
PATH=${patch:location}/bin:${tcl:location}/bin:%(PATH)s
patch-options = -p1
......
......@@ -25,10 +25,10 @@ environment =
PKG_CONFIG_PATH=${libyaml:location}/lib/
[ruby2.6]
[ruby2.7]
<= ruby-common
url = https://ftp.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.xz
md5sum = b8a4e2bdbb76485c3d6690e57be67750
url = https://ftp.ruby-lang.org/pub/ruby/2.7/ruby-2.7.8.tar.xz
md5sum = 27af2c340d0524ab272d564ddfd733d9
[ruby]
<= ruby2.6
<= ruby2.7
......@@ -23,9 +23,21 @@ setup-eggs =
${PyWavelets:egg}
${pillow-python:egg}
networkx
pythran
packaging
rpath =
${openblas:location}/lib
[scikit-image:python2]
setup-eggs =
${numpy:egg}
${scipy:egg}
${cython:egg}
${PyWavelets:egg}
${pillow-python:egg}
networkx
[scikit-image-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
......
......@@ -15,6 +15,7 @@ recipe = zc.recipe.egg:custom
egg = scikit-learn
environment = scikit-learn-env
setup-eggs =
${cython:egg}
${numpy:egg}
${scipy:egg}
rpath =
......
......@@ -36,15 +36,19 @@ init =
zc.buildout.easy_install.default_versions(versions)
# tempstorage5 is plain upstream egg
[tempstorage5]
# tempstorage6 and tempstorage5 are plain upstream egg
[tempstorage6]
recipe = zc.recipe.egg:eggs
egg = tempstorage
eggs = ${:egg}
egg-versions =
tempstorage = 5.2
tempstorage = 6.0.0
[tempstorage5]
<= tempstorage6
egg-versions =
tempstorage = 5.2
# tempstorage4-wc2 is tempstorage 3 + backports for loadBefore fixes
[tempstorage4-wc2]
......
......@@ -26,9 +26,6 @@ md5sum = 51fe2bcbff1bbce77a25d180fd247f7d
pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
pre-configure =
autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability
# XXX workaround path on slaprunner with a double slash
# https://github.com/tesseract-ocr/tesseract/issues/3527
configure-options = --prefix=$(python -c 'print("""@@LOCATION@@""".replace("//", "/"))')
environment =
PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
......
......@@ -42,4 +42,7 @@ inline =
[versions]
freezegun = 1.0.0
freezegun = 1.5.1:whl
[versions:python2]
freezegun = 0.3.15
[python]
part = python2.7
[openssl]
<= openssl-1.1
[buildout]
extends =
../../stack/slapos-py2.cfg
......@@ -15,6 +15,9 @@ extends =
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts =
dcron
logrotate
......@@ -27,12 +30,6 @@ parts =
template-crontab-line
slapos-cookbook
[python]
part = python2.7
[openssl]
<= openssl-1.1
[rssgen-eggs]
recipe = zc.recipe.egg
interpreter = python-${:_buildout_section_name_}
......
......@@ -28,6 +28,9 @@ extends =
../../stack/nxdtest.cfg
./buildout.hash.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts +=
slapos-cookbook
beremiz-eggs
......@@ -41,13 +44,6 @@ parts +=
# Always build GCC for Fortran (see openblas).
max_version = 0
[python]
# Beremiz works with python2.7 for now, the code is not yet upgraded for python3
part = python2.7
[openssl]
<= openssl-1.1
[open62541]
# Beremiz need it to be in folder parts/open62541
# as Beremiz search for open62541 to BEREMIZ_PATH/../open62541
......
......@@ -11,6 +11,9 @@ extends =
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts =
beremiz-source
slapos-cookbook
......@@ -19,12 +22,6 @@ parts =
matiec
open62541
[python]
part = python2.7
[openssl]
<= openssl-1.1
[gcc]
# we want this SR to use a fixed gcc (so that we have the same gcc as open62541 component)
max_version = 0
......
......@@ -3,17 +3,15 @@ extends =
../../stack/slapos.cfg
../../component/vm-img/debian.cfg
../../component/rina-tools/buildout.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts =
slapos-cookbook
template
download-cache = ${:directory}/download-cache
[python]
part = python2.7
[openssl]
<= openssl-1.1
[template]
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
......
......@@ -114,6 +114,7 @@ zope.testrunner = 5.2
recipe = zc.recipe.egg
eggs =
${:recipe}
zc.buildout[test]
zope.testing
zope.testrunner
scripts =
......
......@@ -2,8 +2,5 @@
extends =
software.cfg
[python]
part = python2.7
[openssl]
<= openssl-1.1
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
......@@ -6,6 +6,10 @@ extends =
../../component/manpy/buildout.cfg
../../stack/monitor/buildout.cfg
./buildout.hash.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts =
slapos-cookbook
manpy
......@@ -21,12 +25,6 @@ max_version = 0
# also use old gcc version for old scipy version used in python2
part = gcc-8.5
[python]
part = python2.7
[openssl]
<= openssl-1.1
[dream-repository.git]
revision = f3bcf115741886835df8c0ca0fdbf510d77d8db8
......
......@@ -213,6 +213,7 @@
"type": "object"
}
},
"additionalProperties": false,
"type": "object"
},
"zope-partition-dict": {
......@@ -285,6 +286,7 @@
"type": "object"
}
},
"additionalProperties": false,
"type": "object"
},
"kumofs": {
......
[buildout]
extends =
../../stack/erp5/buildout.cfg
[python]
part = python3
[openssl]
<= openssl-3.0
[erp5]
repository = https://lab.nexedi.com/nexedi/erp5.git
branch = zope4py3
develop = true
[neoppod-repository]
# Pinned to a revision before 6ffafcbd (Fix egg dependencies, 2024-05-17) which
# is incompatible with our versions:
# The requirement ('msgpack<1,>=0.5.6') is not allowed by your [versions] constraint (1.0.5)
revision = c4443632e3541c064f5b43096099f4a8b74cbf58
[template-zope]
link-binary +=
${python3:location}/bin/2to3
software.cfg.json
\ No newline at end of file
[buildout]
extends =
../../stack/erp5/buildout.cfg
../../stack/slapos-py2.cfg
[template-zope]
link-binary +=
${python2.7:location}/bin/2to3
# Test Suite: ERP5.UnitTest-Master ran at 2024/05/29 06:00:29.349755 UTC
......
......@@ -46,10 +46,14 @@ from cryptography.x509.oid import NameOID
from slapos.testing.testcase import ManagedResource, makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
ERP5PY3 = os.environ['SLAPOS_SR_TEST_NAME'] == 'erp5-py3'
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
os.path.join(os.path.dirname(__file__), '..', '..', 'software%s.cfg' % (
'-py3' if ERP5PY3 else ''))),
software_id=os.environ['SLAPOS_SR_TEST_NAME'],
)
setup_module_executed = False
......@@ -191,7 +195,10 @@ def neo(instance_parameter_dict):
class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMeta):
"""ERP5 base test case
"""
__test_matrix__ = matrix((zeo, neo)) # switch between NEO and ZEO mode
if ERP5PY3:
__test_matrix__ = matrix((zeo, )) # TODO: NEO is not yet enabled for py3
else:
__test_matrix__ = matrix((zeo, neo)) # switch between NEO and ZEO mode
@classmethod
def isNEO(cls):
......
......@@ -51,7 +51,7 @@ import urllib3
from slapos.testing.utils import CrontabMixin
import zc.buildout.configparser
from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule
from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
setUpModule # pyflakes
......@@ -1302,60 +1302,80 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
__partition_reference__ = 'n'
__test_matrix__ = matrix((neo,))
def _getCrontabCommand(self, crontab_name: str) -> str:
"""Read a crontab and return the command that is executed.
overloaded to use crontab from neo partition
"""
with open(
os.path.join(
if ERP5PY3:
# NEO is not ready for python3 at this time, this test is here to become
# an unexpected success once it starts working, so that we remember to
# remove this and enable neo in ERP5InstanceTestCase.__test_matrix__
setup_failed_exception = None
@classmethod
def setUpClass(cls):
try:
super().setUpClass()
except BaseException as e:
cls.setup_failed_exception = e
cls.setUp = lambda self: None
cls.tearDownClass = classmethod(lambda cls: None)
@unittest.expectedFailure
def test_neo_py3(self):
self.assertIsNone(self.setup_failed_exception)
else:
def _getCrontabCommand(self, crontab_name: str) -> str:
"""Read a crontab and return the command that is executed.
overloaded to use crontab from neo partition
"""
with open(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'etc',
'cron.d',
crontab_name,
)) as f:
crontab_spec, = f.readlines()
self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
return crontab_spec.split(None, 5)[-1]
def test_log_rotation(self):
# first run to create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
def check_sqlite_log(path):
with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
con.execute('select * from log')
logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'etc',
'cron.d',
crontab_name,
)) as f:
crontab_spec, = f.readlines()
self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
return crontab_spec.split(None, 5)[-1]
def test_log_rotation(self):
# first run to create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
def check_sqlite_log(path):
with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
con.execute('select * from log')
'var',
'log',
f))
logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'var',
'log',
f))
self._executeCrontabAtDate('logrotate', '2050-01-01')
self._executeCrontabAtDate('logrotate', '2050-01-01')
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'srv',
'backup',
'logrotate',
f'{f}-20500101'))
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'srv',
'backup',
'logrotate',
f'{f}-20500101'))
self._executeCrontabAtDate('logrotate', '2050-01-02')
requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
self._executeCrontabAtDate('logrotate', '2050-01-02')
requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'var',
'log',
f))
for f in logfiles:
check_sqlite_log(
os.path.join(
self.getComputerPartitionPath('neo-0'),
'var',
'log',
f))
class TestPassword(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
__partition_reference__ = 'p'
......
......@@ -23,7 +23,7 @@ import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase
from . import setUpModule as _setUpModule
from . import setUpModule as _setUpModule, ERP5PY3
from .test_erp5 import TestPublishedURLIsReachableMixin
......@@ -38,6 +38,8 @@ def setUpModule():
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
if ERP5PY3:
raise unittest.SkipTest("wendelin.core does not support python3 yet")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......
......@@ -14,7 +14,7 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = d1ca30a1b910b6b775f4f95bd91123a6
md5sum = 956ae53af22b551fbb087415e835868b
[watcher]
_update_hash_filename_ = watcher.in
......@@ -30,35 +30,35 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
[gitconfig.in]
_update_hash_filename_ = template/gitconfig.in
md5sum = eb1230fee50067924ba89f4dc6e82fa9
md5sum = c559a24ab6281268b608ed3bccb8e4ce
[gitlab-parameters.cfg]
_update_hash_filename_ = gitlab-parameters.cfg
md5sum = cfda6d959bb90bf0b9c947383f45ce0a
md5sum = 95b18789111ed239146d243e39ffefbe
[gitlab-shell-config.yml.in]
_update_hash_filename_ = template/gitlab-shell-config.yml.in
md5sum = 69e8ed76b06233d11932a5c0ef16f03b
md5sum = 70d394305f4e1482a5c1a673b0762c6a
[gitlab-unicorn-startup.in]
_update_hash_filename_ = gitlab-unicorn-startup.in
md5sum = 705825e6d8c6b37699f1321805d09de3
[gitlab-puma-startup.in]
_update_hash_filename_ = gitlab-puma-startup.in
md5sum = 838209b9246fa86d6a21fef910f17e25
[gitlab.yml.in]
_update_hash_filename_ = template/gitlab.yml.in
md5sum = 673c393e6728a8d82e6b9a44886785a8
md5sum = 6f8df1467a6168bde7d2f5f42abb1012
[gitaly-config.toml.in]
_update_hash_filename_ = template/gitaly-config.toml.in
md5sum = 58e3d5bbda32583d00cd8f44ec0525b0
md5sum = d769ea27820e932c596c35bbbf3f2902
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = b913c4a1f199a87ad71da6d102adffa4
md5sum = 6d8d20ded84622339d49c60b0e61380c
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
md5sum = b8dea5ca4c6f9fc1ca54eb0265e1fdee
md5sum = c8231583d04bf0d3fe2d26230b94d78d
[macrolib.cfg.in]
_update_hash_filename_ = macrolib.cfg.in
......@@ -72,22 +72,18 @@ md5sum = 4980c1571a4dd7753aaa60d065270849
_update_hash_filename_ = template/nginx.conf.in
md5sum = 8c904510eb39dc212204f68f2b81b068
[rack_attack.rb.in]
_update_hash_filename_ = template/rack_attack.rb.in
md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
[resque.yml.in]
_update_hash_filename_ = template/resque.yml.in
md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in]
_update_hash_filename_ = template/smtp_settings.rb.in
md5sum = 4e1ced687a86e4cfff2dde91237e3942
md5sum = b1becd9ec4c2eeefe573af4bb53c9751
[template-gitlab-resiliency-restore.sh.in]
_update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
md5sum = 87f16b4f4a2370acada46b2751ef3366
md5sum = 8ce31a27e814e750dfd38c92a278fb9e
[unicorn.rb.in]
_update_hash_filename_ = template/unicorn.rb.in
md5sum = b4758129a8d0c47b2c3adb10fefb8275
[puma.rb.in]
_update_hash_filename_ = template/puma.rb.in
md5sum = 707c0c713af41518d21724c1be8efe22
......@@ -15,8 +15,11 @@ configuration.external_url = https://lab.example.com
configuration.db_pool = 10
# rack-attack
configuration.rate_limit_requests_per_period = 10
configuration.rate_limit_period = 60
configuration.rack_attack_enable = true
configuration.rack_attack_max_retry = 10
configuration.rack_attack_find_time = 60
configuration.rack_attack_ban_time = 3600
configuration.rack_attack_ip_whitelist = 127.0.0.1
configuration.time_zone = UTC
......@@ -64,8 +67,10 @@ configuration.sidekiq_memory_killer_max_rss = 1000000
# unicorn
configuration.unicorn_worker_timeout = 60
configuration.unicorn_worker_processes = 2
configuration.puma_worker_timeout = 60
configuration.puma_worker_processes = 2
configuration.puma_min_threads = 1
configuration.puma_max_threads = 16
# unicorn advanced
configuration.unicorn_backlog_socket = 1024
......
......@@ -38,6 +38,8 @@ echo "I: PostgreSQL ready." 1>&2
# make sure pg_trgm extension is enabled for gitlab db
psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' || die "pg_trgm setup failed"
psql -c 'CREATE EXTENSION IF NOT EXISTS btree_gist;' || die "btree_gist setup failed"
if echo "$pgtables" | grep -q '^Did not find any relations' ; then
$RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed"
fi
......@@ -70,8 +72,7 @@ $RAKE cache:clear || die "cache:clear failed"
force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed"
# 3. finally exec to unicorn
exec {{ gitlab_unicorn }} \
-E production \
-c {{ unicorn_rb.output }} \
{{ gitlab_work.location }}/config.ru
# 3. finally exec to puma
exec {{ gitlab_puma }} \
-e production \
-C {{ puma_rb.output }}
......@@ -6,6 +6,7 @@ depends_gitfetch =
${go_github.com_pkg_errors:recipe}
${go_lab.nexedi.com_kirr_git-backup:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_golang.org_x_crypto:recipe}
[go_github.com_libgit2_git2go]
......@@ -13,7 +14,7 @@ depends_gitfetch =
go.importpath = github.com/libgit2/git2go
repository = https://github.com/libgit2/git2go.git
# branch 'next' is required by git-backup
revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8
revision = cbca5b82b8c22c08c183a1f44cad4b8b51ba6f25
[go_github.com_pkg_errors]
<= go-git-package
......@@ -25,10 +26,17 @@ revision = v0.8.0-12-g816c908556
<= go-git-package
go.importpath = lab.nexedi.com/kirr/git-backup
repository = https://lab.nexedi.com/kirr/git-backup.git
revision = da754af24da351291c99caa421a103db09e7a4c4
revision = 3327aedfbe903b210366105b8a4f9f85a313a0a3
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 95433de34f
revision = 8299741f
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto.git
revision = 75b288015ac94e66e3d6715fb68a9b41bf046ec2
......@@ -54,7 +54,8 @@ input = inline: gitlab-shell-work*
srv/backup/logrotate/**
etc/service/postgres-start
srv/redis/**
srv/unicorn/unicorn.socket
srv/puma/puma.socket
.cache
output = ${directory:srv}/exporter.exclude
[gitlab-resiliency-restore-script]
......@@ -70,13 +71,14 @@ context =
raw git_location {{ git_location }}
raw bin_directory ${directory:bin}
raw etc_directory ${directory:etc}
raw run_directory ${directory:run}
raw var_directory ${directory:var}
raw postgress_script ${service-postgresql:services}/postgres-start
raw redis_script ${service-redis:wrapper}
raw unicorn_script ${service-unicorn:wrapper-path}
raw puma_script ${service-puma:wrapper-path}
raw sidekiq_script ${service-sidekiq:wrapper-path}
raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git}
raw redis_pid_file ${service-redis:pid-file}
raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid
raw puma_pid_file ${puma:pid}/puma.pid
raw gitlab_work_location ${gitlab-work:location}
raw promise_lab_location ${directory:promise.slow}
......@@ -12,7 +12,7 @@ parts =
# gitlab-<prog>
# ? mailroom
{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %}
{% set gitlab_progv = 'rails rake puma sidekiq puma-startup' .split() %}
{% for prog in gitlab_progv %}
gitlab-{{ prog }}
{% endfor %}
......@@ -23,7 +23,7 @@ parts =
gitlab-shell-work
service-gitlab-workhorse
service-unicorn
service-puma
service-sidekiq
service-nginx
......@@ -51,29 +51,29 @@ offline = true
[worker-processes]
recipe = slapos.recipe.build
unicorn-worker-processes = {{ instance_parameter_dict['configuration.unicorn_worker_processes'] }}
puma-worker-processes = {{ instance_parameter_dict['configuration.puma_worker_processes'] }}
init =
import multiprocessing
worker_count = int(options['unicorn-worker-processes'])
worker_count = int(options['puma-worker-processes'])
if worker_count == 0:
# automatically load all available CPUs
worker_count = multiprocessing.cpu_count() + 1
worker_count = 2 if worker_count < 2 else worker_count
options['unicorn-worker-processes'] = worker_count
options['puma-worker-processes'] = worker_count
options['nginx-worker-processes'] = worker_count -1
[instance-parameter]
{#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert',
'configuration.unicorn_worker_processes', 'configuration.nginx_worker_processes'] %}
'configuration.puma_worker_processes', 'configuration.nginx_worker_processes'] %}
{%- for key, value in instance_parameter_dict.items() -%}
{%- if key not in DROP_KEY_LIST %}
{{ key }} = {{ value }}
{%- endif -%}
{%- endfor %}
# settings for worker processes:
configuration.unicorn_worker_processes = ${worker-processes:unicorn-worker-processes}
configuration.puma_worker_processes = ${worker-processes:puma-worker-processes}
configuration.nginx_worker_processes = ${worker-processes:nginx-worker-processes}
......@@ -186,16 +186,18 @@ mode = 0700
[gitaly-dir]
recipe = slapos.cookbook:mkdirectory
gitaly = ${directory:var}/gitaly
sockets = ${:gitaly}/sockets
internal = ${directory:var}/int
sockets = ${:gitaly}/s
internal = ${:sockets}/int
log = ${directory:log}/gitaly
[gitaly]
socket = ${directory:var}/gitaly.socket
log = ${gitaly-dir:log}
socket = ${gitaly-dir:sockets}/gitaly.socket
logdir = ${gitaly-dir:log}
location = {{ gitaly_location }}
pid = ${directory:run}/gitaly.pid
internal_socket = ${gitaly-dir:internal}
basedir = ${gitaly-dir:gitaly}
num_workers = 2
[gitaly-socket-listening-promise]
<= monitor-promise-base
......@@ -249,7 +251,7 @@ context-extra =
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section unicorn unicorn
section puma puma
section service_redis service-redis
raw redis_binprefix {{ redis_binprefix }}
......@@ -261,6 +263,7 @@ context-extra =
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly
[nginx.conf]
......@@ -288,12 +291,10 @@ context-extra =
import urllib urllib
section gitlab gitlab
section gitlab_shell_work gitlab-shell-work
section gitlab_shell gitlab-shell
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly
[rack_attack.rb]
<= gitlab-etc-template
url = {{ rack_attack_rb_in }}
[resque.yml]
<= gitlab-etc-template
url = {{ resque_yml_in }}
......@@ -306,11 +307,11 @@ url = {{ smtp_settings_rb_in }}
# contains smtp password
mode = 0600
[unicorn.rb]
[puma.rb]
<= gitlab-etc-template
url = {{ unicorn_rb_in }}
url = {{ puma_rb_in }}
context-extra =
section unicorn unicorn
section puma puma
section directory directory
section gitlab_work gitlab-work
......@@ -340,20 +341,20 @@ prog = {{ prog }}
{% endfor %}
[gitlab-unicorn-startup]
[gitlab-puma-startup]
recipe = slapos.recipe.template:jinja2
mode = 0755
url = {{ gitlab_unicorn_startup_in }}
url = {{ gitlab_puma_startup_in }}
output= ${directory:bin}/${:_buildout_section_name_}
context =
raw bash_bin {{ bash_bin }}
raw gitlab_rake ${gitlab-rake:wrapper-path}
raw gitlab_unicorn ${gitlab-unicorn:wrapper-path}
raw gitlab_puma ${gitlab-puma:wrapper-path}
raw psql_bin {{ postgresql_location }}/bin/psql
section pgsql service-postgresql
raw log_dir ${gitlab:log}
raw var_dir ${directory:var}
section unicorn_rb unicorn.rb
section puma_rb puma.rb
section gitlab_work gitlab-work
......@@ -421,14 +422,13 @@ tune-command =
ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
# config/
cd config &&
ln -sf ${unicorn.rb:output} unicorn.rb &&
ln -sf ${puma.rb:output} puma.rb &&
ln -sf ${gitlab.yml:output} gitlab.yml &&
ln -sf ${database.yml:output} database.yml &&
ln -sf ${resque.yml:output} resque.yml &&
ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml &&
# config/initializers/
cd initializers &&
ln -sf ${rack_attack.rb:output} rack_attack.rb &&
ln -sf ${smtp_settings.rb:output} smtp_settings.rb &&
# public/
cd ../../public &&
......@@ -573,11 +573,12 @@ wrapper-path = ${directory:service}/gitlab-workhorse
command-line = {{ gitlab_workhorse }}
-listenNetwork unix
-listenAddr ${gitlab-workhorse:socket}
-authSocket ${unicorn:socket}
-authSocket ${puma:socket}
-documentRoot ${gitlab-work:location}/public
-secretPath ${gitlab-workhorse:secret}
-logFile ${gitlab-workhorse:log}
-repoPath ${gitlab-repo-dir:repositories}
# repoPath is for patched gitlab-workhorse
# -repoPath ${gitlab-repo-dir:repositories}
# NOTE for profiling
# -pprofListenAddr ...
......@@ -606,41 +607,47 @@ config-command = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket} ht
######################
# unicorn worker #
# puma worker #
######################
[unicorn-dir]
[puma-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/unicorn
log = ${directory:log}/unicorn
srv = ${directory:srv}/puma
log = ${directory:log}/puma
pid = ${directory:srv}/pids
[unicorn]
srv = ${unicorn-dir:srv}
log = ${unicorn-dir:log}
socket = ${directory:srv}/unicorn.socket
[puma]
srv = ${puma-dir:srv}
log = ${puma-dir:log}
socket = ${puma-dir:srv}/puma.socket
pid = ${puma-dir:pid}
[service-unicorn]
[service-puma]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/unicorn
# NOTE we perform db setup / migrations as part of unicorn startup.
wrapper-path = ${directory:service}/puma
# NOTE we perform db setup / migrations as part of puma startup.
# Those operations require PG and Redis to be up and running already, that's
# why we do it here. See gitlab-unicorn-startup for details.
command-line = ${gitlab-unicorn-startup:output}
# why we do it here. See gitlab-puma-startup for details.
command-line = ${gitlab-puma-startup:output}
{% if instance_parameter_dict.get('configuration.root-password') -%}
environment =
GITLAB_ROOT_PASSWORD={{ instance_parameter_dict['configuration.root-password'] }}
{% endif %}
depend =
${promise-unicorn:recipe}
${promise-puma:recipe}
${promise-gitlab-app:recipe}
${promise-gitlab-shell:recipe}
${logrotate-entry-unicorn:recipe}
# gitlab is a service "run" under unicorn
${logrotate-entry-puma:recipe}
# gitlab is a service "run" under puma
# gitlab-shell is called by gitlab
# -> associate their logs rotation to here
${logrotate-entry-gitlab:recipe}
[promise-unicorn]
[promise-puma]
<= promise-byurl
config-command = {{ curl_bin }} --unix-socket ${unicorn:socket} http://localhost/
config-command = {{ curl_bin }} --unix-socket ${puma:socket} http://localhost/
[promise-rakebase]
recipe = slapos.cookbook:wrapper
......@@ -662,10 +669,10 @@ command-line = ${:rake} gitlab:gitlab_shell:check
# rake gitlab:repo:check (fsck all repos)
[logrotate-entry-unicorn]
[logrotate-entry-puma]
<= logrotate-entry-base
log = ${unicorn:log}/*.log
name = unicorn
log = ${puma:log}/*.log
name = puma
copytruncate = true
[logrotate-entry-gitlab]
......@@ -682,8 +689,8 @@ copytruncate = true
[logrotate-entry-gitlab-workhorse]
<= logrotate-entry-base
log = ${gitlab-workhorse-dir:log}//*.log
name = gitlab-shell
log = ${gitlab-workhorse-dir:log}/*.log
name = gitlab-workhorse
copytruncate = true
#######################################
......@@ -828,18 +835,17 @@ cron-entries = ${cron:cron-entries}
[service-gitaly]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/gitaly
#command-line = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
command-line = {{ gitaly_location }}/gitaly ${gitaly-config.toml:output}
environment =
PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin
PATH={{ buildout_bin_directory }}:{{ ruby_location }}/bin:/bin:/usr/bin
# 6. on-reinstantiate actions
# NOTE here we only recompile assets. Other on-reinstantiate actions, which
# require pg and redis running, are performed as part of unicorn service -
# right before its startup (see gitlab-unicorn-startup).
# require pg and redis running, are performed as part of puma service -
# right before its startup (see gitlab-puma-startup).
[on-reinstantiate]
recipe = plone.recipe.command
stop-on-error = true
......
......@@ -53,6 +53,7 @@ context =
section instance_parameter_dict slap-configuration
# program binaries
raw buildout_bin_directory ${buildout:bin-directory}
raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle}
......@@ -64,7 +65,7 @@ context =
raw git_location ${git:location}
raw gitaly_location ${gitaly-repository:location}
raw gitlab_export ${gitlab-export:output}
raw gitlab_workhorse ${gowork:bin}/gitlab-workhorse
raw gitlab_workhorse ${gitlab-workhorse:binary}
raw gopath_bin ${gowork:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location}
......@@ -75,8 +76,8 @@ context =
raw nginx_mime_types ${nginx-output:mime}
raw node_bin_location ${nodejs:location}/bin/
raw openssl_bin ${openssl-output:openssl}
raw postgresql_location ${postgresql10:location}
raw redis_binprefix ${redis28:location}/bin
raw postgresql_location ${postgresql:location}
raw redis_binprefix ${redis:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location}
raw watcher ${watcher:output}
......@@ -88,17 +89,16 @@ context =
raw gitconfig_in ${gitconfig.in:target}
raw monitor_template ${monitor2-template:output}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target}
raw gitlab_puma_startup_in ${gitlab-puma-startup.in:target}
raw gitlab_yml_in ${gitlab.yml.in:target}
raw gitaly_config_toml_in ${gitaly-config.toml.in:target}
raw macrolib_cfg_in ${macrolib.cfg.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target}
raw rack_attack_rb_in ${rack_attack.rb.in:target}
raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target}
raw puma_rb_in ${puma.rb.in:target}
$${:context-extra}
context-extra =
......
This diff is collapsed.
......@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}"
# The directory where Gitaly's executables are stored
bin_dir = "{{ gitaly.location }}"
# # Optional: listen on a TCP socket. This is insecure (no authentication)
# # Optional. The directory where Gitaly can create all files required to
# # properly operate at runtime. If not set, Gitaly will create a directory in
# # the global temporary directory. This directory must exist.
runtime_dir = "{{ gitaly.basedir }}"
# # Optional if socket_path is set. TCP address for Gitaly to listen on. This is insecure (unencrypted connection).
# listen_addr = "localhost:9999"
# tls_listen_addr = "localhost:8888
......@@ -33,6 +38,10 @@ internal_socket_dir = "{{ gitaly.internal_socket }}"
bin_path = "{{ git }}"
# catfile_cache_size = 100
# [[git.config]]
# key = fetch.fsckObjects
# value = true
[[storage]]
name = "default"
path = "{{ gitlab.repositories }}"
......@@ -47,9 +56,9 @@ path = "{{ gitlab.repositories }}"
# You can optionally configure Gitaly to output JSON-formatted log messages to stdout
[logging]
# The directory where Gitaly stores extra log files
dir = "{{ gitaly.log }}"
# format = "json"
# format = "json"
dir = "{{ gitaly.logdir }}"
# format = "text"
format = "json"
# # Optional: Set log level to only log entries with that severity or above
# # One of, in order: debug, info, warn, errror, fatal, panic
# # Defaults to "info"
......@@ -79,7 +88,7 @@ dir = "{{ gitaly.location }}/ruby"
# restart_delay = "5m"
#
# # Number of gitaly-ruby worker processes
# num_workers = 2
num_workers = {{ gitaly.num_workers }}
#
# # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc)
# # NOTE: This only affects RPCs that use Rugged.
......@@ -89,7 +98,47 @@ dir = "{{ gitaly.location }}/ruby"
# The directory where gitlab-shell is installed
dir = "{{ gitlab_shell_work.location }}"
[hooks]
custom_hooks_dir = "{{ gitlab_shell_work.location }}/hooks/"
[gitlab]
secret_file = "{{ gitlab_shell.secret }}"
url = "http+unix://{{ urllib.parse.unquote_plus(gitlab_workhorse.socket) }}"
# Only needed if a UNIX socket is used in `url` and GitLab is configured to
# use a relative path (e.g. /gitlab).
# relative_url_root = '/'
[gitlab.http-settings]
# read_timeout = 300
# user = someone
# password = somepass
# ca_file = /etc/ssl/cert.pem
# ca_path = /etc/pki/tls/certs
# self_signed_cert = false
# # You can adjust the concurrency of each RPC endpoint
# [[concurrency]]
# rpc = "/gitaly.RepositoryService/GarbageCollect"
# max_per_repo = 1
# Daily maintenance designates time slots to run daily to optimize and maintain
# enabled storages.
# [daily_maintenance]
# start_hour = 23
# start_minute = 30
# duration = "45m"
# storages = ["default"]
# disabled = false
# [cgroups]
# count = 10
# mountpoint = "/sys/fs/cgroup"
# hierarchy_root = "gitaly"
# [cgroups.memory]
# enabled = true
# limit = 1048576
# [cgroups.cpu]
# enabled = true
# shares = 512
......@@ -12,15 +12,24 @@
[pack]
threads = 1
# Enable packfile bitmaps
[repack]
writeBitmaps = true
# don't allow corrupt/broken objects to go in
# Enable push (advertisePushOptions) options
[receive]
fsckObjects = true
advertisePushOptions = true
[user]
name = {{ cfg('email_display_name') }}
email = {{ cfg('email_from') }}
# Enable fsyncObjectFiles to reduce risk of repository corruption if the server crashes
[core]
autocrlf = input
fsyncObjectFiles = true
[gc]
auto = 0
......@@ -7,13 +7,15 @@
# GitLab user. git by default
user: {{ backend_info.user }}
# Url to gitlab instance. Used for api calls. Should end with a slash.
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(unicorn.socket) }}/"
# URL to GitLab instance, used for API calls. Default: http://localhost:8080.
# For relative URL support read http://doc.gitlab.com/ce/install/relative_url.html
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(puma.socket) }}/"
http_settings:
{# we don't need any
<%= @http_settings.to_json if @http_settings %>
#}
# read_timeout: 300
# user: someone
# password: somepass
# ca_file: /etc/ssl/cert.pem
......@@ -34,35 +36,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
# Default is .gitlab_shell_secret in the root directory.
secret_file: "{{ gitlab_shell.secret }}"
# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
# Default is hooks in the gitlab-shell directory.
custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
# Redis settings used for pushing commit notices to gitlab
redis:
bin: {{ redis_binprefix }}/redis-cli
host: {# <%= @redis_host %> #}
port: {# <%= @redis_port %> #}
socket: {{ service_redis.unixsocket }}
database: {# <%= @redis_database %> #}
namespace: resque:gitlab
# Log file.
# Default is gitlab-shell.log in the root directory.
log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
# Log level. INFO by default
log_level:
log_level: INFO
# Log format. 'text' by default
log_format: text
# Audit usernames.
# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
# incurs an extra API call on every gitlab-shell command.
audit_usernames:
# Enable git-annex support
# git-annex allows managing files with git, without checking the file contents into git
# See https://git-annex.branchable.com/ for documentation
# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
# For CentOS: sudo yum install epel-release && sudo yum install git-annex
git_annex_enabled:
audit_usernames: false
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %}
# # # # # # # # # # # # # # # # # #
# GitLab application config file #
# # # # # # # # # # # # # # # # # #
#
########################### NOTE #####################################
# This file should not receive new settings. All configuration options #
# * are being moved to ApplicationSetting model! #
# If a setting requires an application restart say so in that screen. #
# If you change this file in a merge request, please also create #
# a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests. #
# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md #
########################################################################
#
#
# How to use:
# 1. Copy file as gitlab.yml
# 2. Update gitlab -> host with your fully qualified domain name
# 3. Update gitlab -> email_from
# 4. If you installed Git from source, change git -> bin_path to /usr/local/bin/git
# IMPORTANT: If Git was installed in a different location use that instead.
# You can check with `which git`. If a wrong path of Git is specified, it will
# result in various issues such as failures of GitLab CI builds.
# 5. Review this configuration file for other settings you may want to adjust
production: &base
#
# 1. GitLab app settings
......@@ -18,6 +40,9 @@ production: &base
host: {{ external_url.hostname }}
port: {{ external_url.port or default_port[external_url.scheme] }}
https: {{ cfg_https }}
# The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
# Default is 95% of the worker timeout
max_request_duration_seconds: 57
{# ssh is disabled completely in slapos version
# Uncommment this line below if your ssh host is different from HTTP/HTTPS one
......@@ -55,6 +80,8 @@ production: &base
worker_src: "'self' blob:"
report_uri:
allowed_hosts: []
# Trusted Proxies
# Customize if you have GitLab behind a reverse proxy which is running on a different machine.
# Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
......@@ -122,6 +149,15 @@ production: &base
repository_downloads_path: <%= @gitlab_repository_downloads_path %>
#}
## Impersonation settings
impersonation_enabled: true
## Disable jQuery and CSS animations
# disable_animations: true
## Application settings cache expiry in seconds (default: 60)
# application_settings_cache_seconds: 60
{# we do not support reply by email
## Reply by email
# Allow users to comment on issues and merge requests by replying to notification emails.
......@@ -414,7 +450,9 @@ production: &base
# Gitaly settings
gitaly:
# Default Gitaly authentication token. Can be overriden per storage. Can
# Path to the directory containing Gitaly client executables.
client_path: {{ gitaly.location }}
# Default Gitaly authentication token. Can be overridden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly.
token:
......@@ -463,7 +501,6 @@ production: &base
authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
repos_path: {{ gitlab.repositories }}
hooks_path: {{ gitlab_shell_work.location }}/hooks/
secret_file: {{ gitlab_shell.secret }}
# Git over HTTP
......@@ -483,17 +520,16 @@ production: &base
# gitlab-shell needs to be set to true
git_annex_enabled: <%= @git_annex_enabled %>
workhorse:
# File that contains the secret key for verifying access for gitlab-workhorse.
secret_file: {{ gitlab_workhorse.secret }}
## Git settings
# CAUTION!
# Use the default values unless you really know what you are doing
git:
bin_path: {{ git }}
# The next value is the maximum memory size grit can use
# Given in number of bytes per git object (e.g. a commit)
# This value can be increased if you have very large commits
max_size: {{ cfg('git_max_size') }}
# Git timeout to read a commit, in seconds
timeout: {{ cfg('git_timeout') }}
#
# 5. Extra customization
......@@ -515,11 +551,6 @@ production: &base
<% end %>
#}
{# we are ok (for now) with default rack-attack git settings
rack_attack:
git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %>
#}
## Site ICP License
# XXX unquote needed only for slapos.core earlier than
......@@ -530,6 +561,22 @@ production: &base
{# ICP: '{{ cfg("icp_license") }}' #}
{% endif %}
rack_attack:
git_basic_auth:
# Rack Attack IP banning enabled
enabled: {{ cfg("rack_attack_enable") }}
#
# Whitelist requests from 127.0.0.1 for web proxies (NGINX/Apache) with incorrect headers
ip_whitelist: [{{ cfg("rack_attack_ip_whitelist")}}]
#
# Limit the number of Git HTTP authentication attempts per IP
maxretry: {{ cfg("rack_attack_max_retry") }}
#
# Reset the auth attempt counter per IP after 60 seconds
findtime: {{ cfg("rack_attack_find_time") }}
#
# Ban an IP for one hour (3600s) after too many auth attempts
bantime: {{ cfg("rack_attack_ban_time") }}
development:
<<: *base
......
# see: https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-cookbooks/gitlab/templates/default/puma.rb.erb
{% from 'macrolib.cfg.in' import cfg with context %}
# frozen_string_literal: true
# Load "path" as a rackup file.
#
# The default is "config.ru".
#
rackup 'config.ru'
pidfile '{{ puma.pid }}/puma.pid'
state_path '{{ puma.pid }}/puma.state'
stdout_redirect '{{ puma.log }}/puma.stdout.log',
'{{ puma.log }}/puma.stderr.log',
true
# Configure "min" to be the minimum number of threads to use to answer
# requests and "max" the maximum.
#
# The default is "0, 16".
#
threads {{ cfg("puma_min_threads") }}, {{ cfg("puma_max_threads") }}
# By default, workers accept all requests and queue them to pass to handlers.
# When false, workers accept the number of simultaneous requests configured.
#
# Queueing requests generally improves performance, but can cause deadlocks if
# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
#
# When set to false this may require a reverse proxy to handle slow clients and
# queue requests before they reach puma. This is due to disabling HTTP keepalive
queue_requests false
# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
# accepted protocols.
bind 'unix://{{ puma.socket }}'
directory '{{ gitlab_work.location }}'
workers {{ cfg("puma_worker_processes") }}
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events"
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/puma_worker_killer_initializer"
on_restart do
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
options = { workers: {{ cfg("puma_worker_processes") }} }
before_fork do
# Signal to the puma killer
Gitlab::Cluster::PumaWorkerKillerInitializer.start options unless ENV['DISABLE_PUMA_WORKER_KILLER']
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
end
Gitlab::Cluster::LifecycleEvents.set_puma_options options
on_worker_boot do
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
end
# Preload the application before starting the workers; this conflicts with
# phased restart feature. (off by default)
preload_app!
tag 'gitlab-puma-worker'
# Verifies that all workers have checked in to the master process within
# the given timeout. If not the worker process will be restarted. Default
# value is 60 seconds.
#
worker_timeout {{ cfg("puma_worker_timeout") }}
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#lower-latency-better-throughput
wait_for_less_busy_worker ENV.fetch('PUMA_WAIT_FOR_LESS_BUSY_WORKER', 0.001).to_f
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#nakayoshi_fork
nakayoshi_fork unless ENV['DISABLE_PUMA_NAKAYOSHI_FORK'] == 'true'
# Use json formatter
require_relative "{{ gitlab_work.location }}/lib/gitlab/puma_logging/json_formatter"
json_formatter = Gitlab::PumaLogging::JSONFormatter.new
log_formatter do |str|
json_formatter.call(str)
end
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %}
# 1. Rename this file to rack_attack.rb
# 2. Review the paths_to_be_protected and add any other path you need protecting
#
paths_to_be_protected = [
"#{Rails.application.config.relative_url_root}/users/password",
"#{Rails.application.config.relative_url_root}/users/sign_in",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
"#{Rails.application.config.relative_url_root}/users",
"#{Rails.application.config.relative_url_root}/users/confirmation",
"#{Rails.application.config.relative_url_root}/unsubscribes/"
]
# Create one big regular expression that matches strings starting with any of
# the paths_to_be_protected.
paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
unless Rails.env.test? || !rack_attack_enabled
Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
if req.post? && req.path =~ paths_regex
req.ip
end
end
end
......@@ -21,8 +21,6 @@ if Rails.env.production?
enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }},
# ssl:
openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}'
# ca_path:
# ca_file:
}
end
{% else %}
......
......@@ -21,15 +21,15 @@ redis_pid_file="{{ redis_pid_file }}"
postgres_pid_file="{{ postgres_pid_file }}"
bin_location="{{ bin_directory }}"
run_location="{{ run_directory }}"
git_location="{{ git_location }}"
go_work_bin="{{ go_work_bin }}"
etc_location="{{ etc_directory }}"
gitlab_work="{{ gitlab_work_location }}"
promise_check="{{ promise_lab_location }}"
unicorn_script="{{ unicorn_script }}"
puma_script="{{ puma_script }}"
puma_pid_file="{{ puma_pid_file }}"
sidekiq_script="{{ sidekiq_script }}"
var_location="{{ run_directory }}/.."
var_location="{{ var_directory }}"
# export GIT_EXEC_PATH=$git_location/libexec/git-core/
......@@ -56,7 +56,7 @@ kill_process () {
check_process $postgres_pid_file "Postgres"
check_process $redis_pid_file "Redis"
check_process $run_location/unicorn.pid "Unicorn"
check_process $puma_pid_file "Puma"
if [ -f "$postgres_pid_file" ]; then
rm $postgres_pid_file
......@@ -90,14 +90,14 @@ echo "Checking gitlab promises..."
echo "[info] Not all promises are checked!"
$promise_check/gitlab-app
echo "Starting Unicorn to check gitlab-shell promise..."
$unicorn_script &
unicorn_pid=$!
trap "kill $postgres_pid $redis_pid $unicorn_pid" EXIT TERM INT
echo "Starting Puma to check gitlab-shell promise..."
$puma_script &
puma_pid=$!
trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
sleep 60
if [ -s "$run_location/unicorn.pid" ]; then
unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $unicorn_ppid" EXIT TERM INT
if [ -s "$puma_pid_file" ]; then
puma_pid=$(head -n 1 $puma_pid_file) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
fi
$promise_check/gitlab-shell
......@@ -109,7 +109,7 @@ $promise_check/gitlab-shell
kill_process $postgres_pid
kill_process $redis_pid
kill_process $unicorn_pid
kill_process $puma_pid
RESTORE_EXIT_CODE=$?
......
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg with context %}
# What ports/sockets to listen on, and what options for them.
# we listen only on unix socket
listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
#listen "127.0.0.1:8888", :tcp_nopush => true
working_directory '{{ gitlab_work.location }}'
# What the timeout for killing busy workers is, in seconds
timeout {{ cfg('unicorn_worker_timeout') }}
# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
preload_app true
# Enable this flag to have unicorn test client connections by writing the
# beginning of the HTTP headers before calling the application. This
# prevents calling the application for connections that have disconnected
# while queued. This is only guaranteed to detect clients on the same
# host unicorn runs on, and unlikely to detect disconnects even on a
# fast LAN.
check_client_connection false
require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
before_exec do |server|
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
# How many worker processes
worker_processes {{ cfg('unicorn_worker_processes') }}
# about before_fork / after_fork - see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
# http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
# What to do before we fork a worker
before_fork do |server, worker|
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
# twice as many worker_processes as you have configured.
#
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
#
# Throttle the master from forking too quickly by sleeping. Due
# to the implementation of standard Unix signal handlers, this
# helps (but does not completely) prevent identical, repeated signals
# from being lost when the receiving process is busy.
# sleep 1
end
# What to do after we fork a worker
after_fork do |server, worker|
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
end
# Where to drop a pidfile
pid '{{ directory.run }}/unicorn.pid'
# Where stderr gets logged
stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
# Where stdout gets logged
stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
{# we do not support Relative url
<%- if @relative_url %>
# Relative url from where GitLab is served
ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
<%- end %>
#}
# Min memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
# Max memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s
......@@ -46,6 +46,7 @@ setup(
'erp5.util',
'supervisor',
'requests',
'beautifulsoup4'
],
zip_safe=True,
test_suite='test',
......
......@@ -26,10 +26,10 @@
##############################################################################
import os
import logging
from urllib.parse import urlparse
import requests
import functools
import bs4
from urllib.parse import urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -46,6 +46,10 @@ class TestGitlab(SlapOSInstanceTestCase):
def getInstanceSoftwareType(cls):
return 'gitlab'
@classmethod
def getInstanceParameterDict(cls):
return {'root-password': 'admin1234'}
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend_url']
......@@ -54,3 +58,33 @@ class TestGitlab(SlapOSInstanceTestCase):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
def test_rack_attack_sign_in_rate_limiting(self):
session = requests.session()
# Load the login page to get a CSRF token.
response = session.get(urljoin(self.backend_url, 'users/sign_in'), verify=False)
self.assertEqual(response.status_code, 200)
# Extract the CSRF token and param.
bsoup = bs4.BeautifulSoup(response.text, 'html.parser')
csrf_param = bsoup.find('meta', dict(name='csrf-param'))['content']
csrf_token = bsoup.find('meta', dict(name='csrf-token'))['content']
request_data = {
'user[login]': 'test',
'user[password]': 'random',
csrf_param: csrf_token}
sign_in = functools.partial(
session.post,
response.url,
data=request_data,
verify=False)
for _ in range(10):
sign_in(headers={'X-Forwarded-For': '1.2.3.4'})
# after 10 authentication failures, this client is rate limited
self.assertEqual(sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).status_code, 429)
# but other clients are not
self.assertNotEqual(sign_in(headers={'X-Forwarded-For': '5.6.7.8'}).status_code, 429)
......@@ -2,16 +2,14 @@
extends =
../../stack/slapos.cfg
../../component/rina-tools/buildout.cfg
# Python2 versions for buildout (keep last)
../../stack/slapos-py2.cfg
parts =
slapos-cookbook
template
[python]
part = python2.7
[openssl]
<= openssl-1.1
[file]
# For old GCC like 4.9.2 on Debian 8.
# XXX: This should be moved to component/rina-tools/buildout.cfg, next to where
......
......@@ -19,11 +19,11 @@ md5sum = 5f39952f94095b1f12f41db76867e71e
[instance-jupyter]
filename = instance-jupyter.cfg.in
md5sum = f9a0e5a134456d74ca8b4d87862f903d
md5sum = 1812fa797b9eb687a634ebe96134b504
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 089e4c511a3c7b110471bf41ca2695a4
md5sum = 6c03113fb53d6ba98476f3353c083984
[erp5-kernel]
filename = ERP5kernel.py
......
......@@ -73,7 +73,7 @@ key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/jupyter-lab
{{ bin_directory }}/jupyter-notebook
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
......@@ -87,16 +87,16 @@ environment =
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
JUPYTERLAB_DIR=${directory:jupyterlab-dir}
LANG=C.UTF-8
[jupyter-password]
recipe = slapos.cookbook:generate.password
bytes = 10
[jupyter-notebook-config]
recipe = slapos.recipe.template:jinja2
url = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
output = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
output = ${directory:jupyter_config_dir}/jupyter_server_config.py
context =
key password jupyter-password:passwd
raw gcc_location {{ gcc_location }}
......@@ -119,6 +119,7 @@ jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
jupyterlab-dir = ${:jupyter_dir}/lab
[request-slave-frontend-base]
recipe = slapos.cookbook:requestoptional
......@@ -194,7 +195,7 @@ output = ${directory:erp5_kernel_dir}/kernel.json
context =
raw python_executable {{ python_executable }}
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment