Skip to content

slapos
slapos
Commit b51f5523 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Merge branch 'master' into x/lte-multiru 

* master:
  golang: Switch default to Go1.21
  wendelin.core: v↑ 2.0.alpha3..post7 -> 2.0.alpha3.post9
  golang: Add go version 1.21
  component/haproxy: build with prometheus exporter
  component/gdb: version up 14.1
  software/erp5/test: move Caucase helper classes in __init__
  software/erp5/test: use type annotations instead of type comments
  chromium,firefox,seleniumserver: update chromium and firefox versions
  software/seleniumserver: configure fontconfig
  component/fontconfig: version up 2.14.2
  cloudoo/test: remove some useless font test
parents 0e90e7d4 1e10b3e7
Hide whitespace changes
Inline Side-by-side
Showing with 472 additions and 469 deletions
component/chromedriver/buildout.cfg
View file @ b51f5523
......@@ -5,21 +5,24 @@
[buildout]
extends =
../chromium/buildout.cfg
../nss/buildout.cfg
../glib/buildout.cfg
../nspr/buildout.cfg
../nss/buildout.cfg
../pcre2/buildout.cfg
../xorg/buildout.cfg
parts =
chromedriver-wrapper
[chromedriver-wrapper-120]
<= chromedriver-wrapper
part = chromedriver-120
[chromedriver-wrapper-91]
<= chromedriver-wrapper
part = chromedriver-91
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
part = chromedriver-2.41
[chromedriver-wrapper]
# generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
......@@ -42,30 +45,36 @@ install =
[chromedriver]
<= chromedriver-91
<= chromedriver-120
[chromedriver-2.41]
[chromedriver-120]
<= chromedriver-download
version = 2.41
# Supports Chrome v67-69
md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
version = 120.0.6099.109
revision-x86_64 = 1217362
generation-x86_64 = 1698717838856458
md5sum-x86_64 = 5cb8d386f01052cfc58c80ec63477db0
[chromedriver-91]
<= chromedriver-download
url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip
version = 91.0.4472.101
# Supports Chrome v91
md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
# This chromedriver is not usable directly, it needs a wrapper.
recipe = slapos.recipe.build:download-unpacked
url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_${:_url}.zip
library =
${nss:location}/lib
${nspr:location}/lib
${glib:location}/lib
${libX11:location}/lib
${libXau:location}/lib
${libxcb:location}/lib
${libXdmcp:location}/lib
${nspr:location}/lib
${nss:location}/lib
${pcre2:location}/lib
[chromedriver-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux64
url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision-x86_64}%2Fchromedriver_linux64.zip?generation=${:generation-x86_64}&alt=media
md5sum = ${:md5sum-x86_64}
component/chromium/buildout.cfg
View file @ b51f5523
......@@ -65,36 +65,35 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[chromium-wrapper-120]
<= chromium-wrapper
part = chromium-120
[chromium-wrapper-91]
<= chromium-wrapper
part = chromium-91
[chromium-wrapper-69]
<= chromium-wrapper
part = chromium-69
[chromium]
<= chromium-120
[chromium-120]
<= chromium-download
version = 120.0.6099.109
revision-x86_64 = 1217362
md5sum-x86_64 = 86719e40f3d33f1b421d073bb4a71f41
generation-x86_64 = 1698717835110888
[chromium]
<= chromium-91
[chromium-91]
<= chromium-download
version = 91.0.4472.114
revision_x86-64 = 870763
revision-x86_64 = 870763
md5sum-x86_64 = 74eab41580469c2b8117cf396db823cb
generation-x86_64 = 1617926496067901
[chromium-69]
<= chromium-download
version = 69.0.3497.0
revision_x86-64 = 576753
md5sum-x86_64 = 08ac27fd40ace4ca8dfbd1db403deccb
generation-x86_64 = 1532051976706023
[chromium-download]
# macro to download a binary build of chromium and generate a
# wrapper as chrome-slapos in the part directory
......@@ -148,5 +147,5 @@ path =
${fontconfig:location}/bin
[chromium-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision-x86_64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
md5sum = ${:md5sum-x86_64}
component/firefox/buildout.cfg
View file @ b51f5523
......@@ -61,17 +61,9 @@ install =
<= firefox-wrapper
part = firefox-115
[firefox-wrapper-78]
[firefox-wrapper-102]
<= firefox-wrapper
part = firefox-78
[firefox-wrapper-68]
<= firefox-wrapper
part = firefox-68
[firefox-wrapper-60]
<= firefox-wrapper
part = firefox-60
part = firefox-102
[firefox-default-fonts-conf]
recipe = slapos.recipe.template:jinja2
......@@ -103,23 +95,12 @@ version = 115.3.1esr
i686-md5sum = f0df1b5cce1edd65addc823da02f9488
x86_64-md5sum = 910c0786459cf1e4dc214e6402d0633e
[firefox-78]
[firefox-102]
<= firefox-download
version = 78.1.0esr
i686-md5sum = 09595a1b9a99d17a618a51bc1f971e5e
x86_64-md5sum = 06f4d488721ce7229d9a86cb4c6786f3
version = 102.15.1esr
i686-md5sum = 418b51b3553e98070998fcdbc344487d
x86_64-md5sum = ff477480d34e44fbd0040c32ed905aaf
[firefox-68]
<= firefox-download
version = 68.0.2esr
i686-md5sum = eaa9e0246eb2a31ccf55c100dc2edd5a
x86_64-md5sum = d22dc17ce0949cdff78009afca6f2043
[firefox-60]
<= firefox-download
version = 60.0.2esr
i686-md5sum = ce7c80716036dfb5c2fb1ca2538556ff
x86_64-md5sum = 6fe25d9a3fcc82670320242c9047d1da
[firefox-download]
recipe = slapos.recipe.build
......@@ -199,18 +180,6 @@ version = 0.33.0
i686-md5sum = c4a9e6c92dc493f25c8d390f1c6fb11c
x86_64-md5sum = 563c82cfbb21478450e1c828e3730b10
[geckodriver-0.24.0]
<= geckodriver-base
version = 0.24.0
i686-md5sum = b88eee754f6c90b01f760f7a453dda95
x86_64-md5sum = 7552b85e43973c84763e212af7cca566
[geckodriver-0.22.0]
<= geckodriver-base
version = 0.22.0
i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
[geckodriver-base]
# Installs geckodriver ${version}
recipe = slapos.recipe.build
......
component/fontconfig/buildout.cfg
View file @ b51f5523
......@@ -3,10 +3,11 @@ extends =
../freetype/buildout.cfg
../libxml2/buildout.cfg
../pkgconfig/buildout.cfg
../python3/buildout.cfg
../bzip2/buildout.cfg
../zlib/buildout.cfg
../bzip2/buildout.cfg
../gperf/buildout.cfg
../xz-utils/buildout.cfg
buildout.hash.cfg
parts =
......@@ -15,8 +16,8 @@ parts =
[fontconfig]
recipe = slapos.recipe.cmmi
shared = true
url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
md5sum = 733f5e2371ca77b69707bd7b30cc2163
url = https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.14.2.tar.xz
md5sum = 95261910ea727b5dd116b06fbfd84b1f
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -24,7 +25,8 @@ configure-options =
--enable-libxml2
--with-add-fonts=no
environment =
PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
PATH=${python3:location}/bin:${pkgconfig:location}/bin:${gperf:location}/bin:${xz-utils:location}/bin:%(PATH)s
PYTHON=${python3:location}/bin/python3
PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
......
component/gdb/buildout.cfg
View file @ b51f5523
......@@ -13,8 +13,8 @@ parts = gdb
[gdb]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/gdb/gdb-9.2.tar.xz
md5sum = db95524e554870209ab7d9f8fd8dc557
url = https://ftp.gnu.org/gnu/gdb/gdb-14.1.tar.xz
md5sum = 4a084d03915b271f67e9b8ea2ab24972
location = @@LOCATION@@
# gdb refuses to build in-tree -> build it inside build/
pre-configure =
......
component/golang/buildout.cfg
View file @ b51f5523
......@@ -30,24 +30,28 @@ environment =
PATH=${swig:location}/bin:${patch:location}/bin:%(PATH)s
GOROOT_FINAL=${:location}
${:environment-extra}
patch-options = -p1
[golang-common-pre-1.21]
<= golang-common
# TestChown currently fails in a user-namespace
# https://github.com/golang/go/issues/42525
# the patches apply to go >= 1.12
patch-options = -p1
patches =
# the patches apply to 1.21 > go >= 1.12
# (in go 1.21 we can't apply it, due to code changes,
# in go > 1.21 it's hopefully fixed with
# https://github.com/golang/go/commit/9f03e8367d85d75675b2f2e90873e3293799d8aa)
patches +=
${:_profile_base_location_}/skip-chown-tests.patch#d4e3c8ef83788fb2a5d80dd75034786f
[golang-common-pre-1.19]
<= golang-common
<= golang-common-pre-1.21
# TestSCMCredentials fails in a user-namespace if golang version < 1.19
# https://github.com/golang/go/issues/42525
patches +=
${:_profile_base_location_}/fix-TestSCMCredentials.patch#1d8dbc97cd579e03fafd8627d48f1c59
[golang14]
<= golang-common-pre-1.19
# https://golang.org/doc/install/source#bootstrapFromSource
......@@ -117,13 +121,28 @@ environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.20]
<= golang-common
<= golang-common-pre-1.21
url = https://go.dev/dl/go1.20.6.src.tar.gz
md5sum = 1dc2d18790cfaede7df1e73a1eff8b7b
# go1.20 requires go1.17.13 to bootstrap (see https://go.dev/doc/go1.20#bootstrap)
environment-extra =
GOROOT_BOOTSTRAP=${golang1.17:location}
[golang1.21]
<= golang-common
url = https://go.dev/dl/go1.21.5.src.tar.gz
md5sum = 99385ded31906f1554c27015bbbee52d
# go1.21 requires go1.17.13 to bootstrap (see https://go.dev/blog/rebuild)
environment-extra =
GOROOT_BOOTSTRAP=${golang1.17:location}
patches +=
# TestChown fix, old fix doesn't work due to code change,
# for golang > 1.21 this patch is hopefully already included with
# https://github.com/golang/go/commit/9f03e8367d85d75675b2f2e90873e3293799d8aa
${:_profile_base_location_}/os-skip-Chown-tests-for-auxiliary-groups-that-fail-d.patch#81b7f75786d9024049c26d1663b79ba4
${:_profile_base_location_}/skip-unshare-mount-test.patch#325446d5135452e8685e95ab99c13a51
# ---- infrastructure to build Go workspaces / projects ----
# gowork is the top-level section that defines Go workspace.
......@@ -183,7 +202,7 @@ bin = ${gowork.dir:bin}
depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
golang = ${golang1.20:location}
golang = ${golang1.21:location}
# no special build flags by default
buildflags =
......
component/golang/os-skip-Chown-tests-for-auxiliary-groups-that-fail-d.patch 0 → 100644
View file @ b51f5523
From 9f03e8367d85d75675b2f2e90873e3293799d8aa Mon Sep 17 00:00:00 2001
From: "Bryan C. Mills" <bcmills@google.com>
Date: Tue, 15 Aug 2023 18:01:16 -0400
Subject: [PATCH] os: skip Chown tests for auxiliary groups that fail due to
permission errors
This addresses the failure mode described in
https://git.alpinelinux.org/aports/commit/community/go/tests-filter-overflow-gid.patch?id=9851dde0f5d2a5a50f7f3b5323d1b2ff22e1d028,
but without special-casing an implementation-specific group ID.
For #62053.
Change-Id: I70b1046837b8146889fff7085497213349cd2bf0
Reviewed-on: https://go-review.googlesource.com/c/go/+/520055
Reviewed-by: Ian Lance Taylor <iant@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Bryan Mills <bcmills@google.com>
Auto-Submit: Bryan Mills <bcmills@google.com>
---
src/os/os_unix_test.go | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/src/os/os_unix_test.go b/src/os/os_unix_test.go
index 9041b25471..e4271ff905 100644
--- a/src/os/os_unix_test.go
+++ b/src/os/os_unix_test.go
@@ -75,6 +75,12 @@ func TestChown(t *testing.T) {
t.Log("groups: ", groups)
for _, g := range groups {
if err = Chown(f.Name(), -1, g); err != nil {
+ if testenv.SyscallIsNotSupported(err) {
+ t.Logf("chown %s -1 %d: %s (error ignored)", f.Name(), g, err)
+ // Since the Chown call failed, the file should be unmodified.
+ checkUidGid(t, f.Name(), int(sys.Uid), gid)
+ continue
+ }
t.Fatalf("chown %s -1 %d: %s", f.Name(), g, err)
}
checkUidGid(t, f.Name(), int(sys.Uid), g)
@@ -123,6 +129,12 @@ func TestFileChown(t *testing.T) {
t.Log("groups: ", groups)
for _, g := range groups {
if err = f.Chown(-1, g); err != nil {
+ if testenv.SyscallIsNotSupported(err) {
+ t.Logf("chown %s -1 %d: %s (error ignored)", f.Name(), g, err)
+ // Since the Chown call failed, the file should be unmodified.
+ checkUidGid(t, f.Name(), int(sys.Uid), gid)
+ continue
+ }
t.Fatalf("fchown %s -1 %d: %s", f.Name(), g, err)
}
checkUidGid(t, f.Name(), int(sys.Uid), g)
@@ -181,12 +193,22 @@ func TestLchown(t *testing.T) {
t.Log("groups: ", groups)
for _, g := range groups {
if err = Lchown(linkname, -1, g); err != nil {
+ if testenv.SyscallIsNotSupported(err) {
+ t.Logf("lchown %s -1 %d: %s (error ignored)", f.Name(), g, err)
+ // Since the Lchown call failed, the file should be unmodified.
+ checkUidGid(t, f.Name(), int(sys.Uid), gid)
+ continue
+ }
t.Fatalf("lchown %s -1 %d: %s", linkname, g, err)
}
checkUidGid(t, linkname, int(sys.Uid), g)
// Check that link target's gid is unchanged.
checkUidGid(t, f.Name(), int(sys.Uid), int(sys.Gid))
+
+ if err = Lchown(linkname, -1, gid); err != nil {
+ t.Fatalf("lchown %s -1 %d: %s", f.Name(), gid, err)
+ }
}
}
--
2.36.0
component/golang/skip-unshare-mount-test.patch 0 → 100644
View file @ b51f5523
https://github.com/golang/go/commit/092671423cd95eaa6df93eb29442fef41504d097 breaks
TestUnshareMountNameSpace test on SlapOS.
---
diff --git a/src/syscall/exec_linux_test.go b/src/syscall/exec_linux_test.go
index f4ff7bf81b..bc8bdb0a35 100644
--- a/src/syscall/exec_linux_test.go
+++ b/src/syscall/exec_linux_test.go
@@ -206,6 +206,7 @@ func TestGroupCleanupUserNamespace(t *testing.T) {
// Test for https://go.dev/issue/19661: unshare fails because systemd
// has forced / to be shared
func TestUnshareMountNameSpace(t *testing.T) {
+ t.Skip("skipping: not supported in SlapOS")
testenv.MustHaveExec(t)
if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" {
component/haproxy/buildout.cfg
View file @ b51f5523
......@@ -47,6 +47,7 @@ make-options =
${:QUIC}
USE_PCRE=1
USE_ZLIB=1
USE_PROMEX=1
ZLIB_INC=${zlib:location}/include
ZLIB_LIB=${zlib:location}/lib
ADDLIB="${:SSL_ADDLIB} -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
......
component/wendelin.core/buildout.cfg
View file @ b51f5523
......@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/wendelin.core.git
branch = master
revision = wendelin.core-2.0.alpha3-7-g885b355
revision = wendelin.core-2.0.alpha3-9-gda765ef
# dir is pretty name as top-level recipe
location = ${buildout:parts-directory}/wendelin.core
git-executable = ${git:location}/bin/git
software/cloudooo/test/test.py
View file @ b51f5523
......@@ -151,7 +151,6 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'Courier New': 'LiberationSans',
'DejaVu Sans': 'DejaVuSans',
'DejaVu Sans Condensed': 'LiberationSans',
'DejaVu Sans ExtraLight': 'LiberationSans',
'DejaVu Sans Mono': 'DejaVuSansMono',
'DejaVu Serif': 'DejaVuSerif',
'DejaVu Serif Condensed': 'LiberationSans',
......@@ -168,11 +167,11 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LiberationSans',
'OpenSymbol': {'DejaVuSans', 'OpenSymbol'},
'OpenSymbol': {'NotoSans-Regular', 'OpenSymbol'},
'Palatino': 'LiberationSans',
'Roboto Black': 'LiberationSans',
'Roboto Condensed Light': 'LiberationSans',
'Roboto Condensed Regular': 'LiberationSans',
'Roboto Condensed': 'RobotoCondensed-Regular',
'Roboto Light': 'LiberationSans',
'Roboto Medium': 'LiberationSans',
'Roboto Thin': 'LiberationSans',
......@@ -201,43 +200,42 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
pdf_producer = 'LibreOffice 7.5'
expected_font_mapping = {
'Arial': 'LiberationSans',
'Arial Black': 'DejaVuSans',
'Avant Garde': 'DejaVuSans',
'Bookman': 'DejaVuSans',
'Arial Black': 'NotoSans-Regular',
'Avant Garde': 'NotoSans-Regular',
'Bookman': 'NotoSans-Regular',
'Carlito': 'Carlito',
'Comic Sans MS': 'DejaVuSans',
'Comic Sans MS': 'NotoSans-Regular',
'Courier New': 'LiberationMono',
'DejaVu Sans': 'DejaVuSans',
'DejaVu Sans Condensed': 'DejaVuSansCondensed',
'DejaVu Sans ExtraLight': 'DejaVuSans',
'DejaVu Sans Mono': 'DejaVuSansMono',
'DejaVu Serif': 'DejaVuSerif',
'DejaVu Serif Condensed': 'DejaVuSerifCondensed',
'Garamond': 'DejaVuSerif',
'Garamond': 'NotoSerif-Regular',
'Gentium Basic': 'GentiumBasic',
'Gentium Book Basic': 'GentiumBookBasic',
'Georgia': 'DejaVuSerif',
'Georgia': 'NotoSerif-Regular',
'Helvetica': 'LiberationSans',
'IPAex Gothic': 'IPAexGothic',
'IPAex Mincho': 'IPAexMincho',
'Impact': 'DejaVuSans',
'Impact': 'NotoSans-Regular',
'Liberation Mono': 'LiberationMono',
'Liberation Sans': 'LiberationSans',
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LinuxLibertineG',
'OpenSymbol': {'OpenSymbol', 'IPAMincho'},
'Palatino': 'DejaVuSerif',
'Palatino': 'NotoSerif-Regular',
'Roboto Black': 'Roboto-Black',
'Roboto Condensed Light': 'RobotoCondensed-Light',
'Roboto Condensed Regular': 'DejaVuSans',
'Roboto Condensed': 'RobotoCondensed-Regular',
'Roboto Light': 'Roboto-Light',
'Roboto Medium': 'Roboto-Medium',
'Roboto Thin': 'Roboto-Thin',
'Times New Roman': 'LiberationSerif',
'Trebuchet MS': 'DejaVuSans',
'Verdana': 'DejaVuSans',
'ZZZdefault fonts when no match': 'DejaVuSans'
'Trebuchet MS': 'NotoSans-Regular',
'Verdana': 'NotoSans-Regular',
'ZZZdefault fonts when no match': 'NotoSans-Regular'
}
def _convert_html_to_pdf(self, src_html):
......
software/erp5/test/test/__init__.py
View file @ b51f5523
......@@ -25,12 +25,26 @@
#
##############################################################################
import hashlib
import itertools
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
import urllib
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
import requests
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import ManagedResource, makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -200,3 +214,175 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMet
def getComputerPartitionPath(cls, partition_reference):
partition_id = cls.getComputerPartition(partition_reference).getId()
return os.path.join(cls.slap._instance_root, partition_id)
class CaucaseService(ManagedResource):
"""A caucase service.
"""
url: str = None
directory: str = None
_caucased_process: subprocess.Popen = None
def open(self) -> None:
# start a caucased and server certificate.
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
self.directory = tempfile.mkdtemp()
caucased_dir = os.path.join(self.directory, 'caucased')
os.mkdir(caucased_dir)
os.mkdir(os.path.join(caucased_dir, 'user'))
os.mkdir(os.path.join(caucased_dir, 'service'))
backend_caucased_netloc = f'{self._cls._ipv4_address}:{findFreeTCPPort(self._cls._ipv4_address)}'
self.url = 'http://' + backend_caucased_netloc
self._caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
# capture subprocess output not to pollute test's own stdout
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(30):
try:
if requests.get(self.url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError('caucased failed to start.')
def close(self) -> None:
self._caucased_process.terminate()
self._caucased_process.wait()
self._caucased_process.stdout.close()
shutil.rmtree(self.directory)
@property
def ca_crt_path(self) -> str:
"""Path of the CA certificate from this caucase.
"""
ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
if not os.path.exists(ca_crt_path):
with open(ca_crt_path, 'w') as f:
f.write(
requests.get(urllib.parse.urljoin(
self.url,
'/cas/crt/ca.crt.pem',
)).text)
return ca_crt_path
class CaucaseCertificate(ManagedResource):
"""A certificate signed by a caucase service.
"""
ca_crt_file: str = None
crl_file: str = None
csr_file: str = None
cert_file: str = None
key_file: str = None
def open(self) -> None:
self.tmpdir = tempfile.mkdtemp()
self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
self.key_file = os.path.join(self.tmpdir, 'key.pem')
def close(self) -> None:
shutil.rmtree(self.tmpdir)
@property
def _caucase_path(self) -> str:
"""path of caucase executable.
"""
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name: str, caucase: CaucaseService) -> None:
"""Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance.
"""
cas_args = [
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
]
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(self.key_file, 'wb') as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(
NameOID.COMMON_NAME,
common_name,
),
])).sign(
key,
hashes.SHA256(),
default_backend(),
)
with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
csr_id = subprocess.check_output(
cas_args + [
'--send-csr', self.csr_file,
],
).split()[0].decode()
assert csr_id
for _ in range(30):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, self.cert_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as cert_file:
assert 'BEGIN CERTIFICATE' in cert_file.read()
def revoke(self, caucase: CaucaseService) -> None:
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
'--revoke-crt', self.cert_file, self.key_file,
])
software/erp5/test/test/test_balancer.py
View file @ b51f5523
import glob
import hashlib
import json
import logging
import os
import re
import shutil
import subprocess
import tempfile
import time
import urllib.parse
from http.server import BaseHTTPRequestHandler
......@@ -15,16 +12,10 @@ from unittest import mock
import pexpect
import psutil
import requests
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import ManagedResource
from slapos.testing.utils import CrontabMixin, ManagedHTTPServer, findFreeTCPPort
from slapos.testing.utils import CrontabMixin, ManagedHTTPServer
from . import ERP5InstanceTestCase, default, matrix, setUpModule
from . import CaucaseCertificate, CaucaseService, ERP5InstanceTestCase, default, matrix, setUpModule
setUpModule # pyflakes
......@@ -34,8 +25,7 @@ class EchoHTTPServer(ManagedHTTPServer):
encoded in json.
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = json.dumps(
......@@ -57,8 +47,7 @@ class EchoHTTP11Server(ManagedHTTPServer):
"""
class RequestHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def do_GET(self):
# type: () -> None
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = json.dumps(
......@@ -75,61 +64,6 @@ class EchoHTTP11Server(ManagedHTTPServer):
log_message = logging.getLogger(__name__ + '.EchoHTTP11Server').info
class CaucaseService(ManagedResource):
"""A caucase service.
"""
url = None # type: str
directory = None # type: str
_caucased_process = None # type: subprocess.Popen
def open(self):
# type: () -> None
# start a caucased and server certificate.
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
self.directory = tempfile.mkdtemp()
caucased_dir = os.path.join(self.directory, 'caucased')
os.mkdir(caucased_dir)
os.mkdir(os.path.join(caucased_dir, 'user'))
os.mkdir(os.path.join(caucased_dir, 'service'))
backend_caucased_netloc = f'{self._cls._ipv4_address}:{findFreeTCPPort(self._cls._ipv4_address)}'
self.url = 'http://' + backend_caucased_netloc
self._caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
# capture subprocess output not to pollute test's own stdout
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(30):
try:
if requests.get(self.url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError('caucased failed to start.')
def close(self):
# type: () -> None
self._caucased_process.terminate()
self._caucased_process.wait()
self._caucased_process.stdout.close()
shutil.rmtree(self.directory)
class BalancerTestCase(ERP5InstanceTestCase):
# We explicitly specify 'balancer' as our software type here,
# therefore we don't request ZODB. We therefore don't
......@@ -143,8 +77,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
return 'balancer'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
return {
'tcpv4-port': 8000,
'computer-memory-percent-threshold': 100,
......@@ -179,12 +112,10 @@ class BalancerTestCase(ERP5InstanceTestCase):
}
@classmethod
def getInstanceParameterDict(cls):
# type: () -> dict
def getInstanceParameterDict(cls) -> dict:
return {'_': json.dumps(cls._getInstanceParameterDict())}
def setUp(self):
# type: () -> None
def setUp(self) -> None:
self.default_balancer_url = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])['default']
......@@ -195,8 +126,7 @@ class SlowHTTPServer(ManagedHTTPServer):
Timeout is 2 seconds by default, and can be specified in the path of the URL
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "text/plain")
timeout = 2
......@@ -214,8 +144,7 @@ class SlowHTTPServer(ManagedHTTPServer):
class TestTimeout(BalancerTestCase, CrontabMixin):
__partition_reference__ = 't'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
# use a slow server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
......@@ -223,8 +152,7 @@ class TestTimeout(BalancerTestCase, CrontabMixin):
parameter_dict['timeout-dict'] = {'default': 1}
return parameter_dict
def test_timeout(self):
# type: () -> None
def test_timeout(self) -> None:
self.assertEqual(
requests.get(
urllib.parse.urljoin(self.default_balancer_url, '/1'),
......@@ -242,15 +170,13 @@ class TestLog(BalancerTestCase, CrontabMixin):
"""
__partition_reference__ = 'l'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
# use a slow server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
return parameter_dict
def test_access_log_format(self):
# type: () -> None
def test_access_log_format(self) -> None:
requests.get(
urllib.parse.urljoin(self.default_balancer_url, '/url_path'),
verify=False,
......@@ -274,8 +200,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
self.assertGreater(request_time, 2 * 1000)
self.assertLess(request_time, 20 * 1000)
def test_access_log_apachedex_report(self):
# type: () -> None
def test_access_log_apachedex_report(self) -> None:
# make a request so that we have something in the logs
requests.get(self.default_balancer_url, verify=False)
......@@ -297,8 +222,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
# having this table means that apachedex could parse some lines.
self.assertIn('<h2>Hits per status code</h2>', report_text)
def test_access_log_rotation(self):
# type: () -> None
def test_access_log_rotation(self) -> None:
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
......@@ -324,8 +248,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
def test_error_log(self):
# type: () -> None
def test_error_log(self) -> None:
# stop backend server
backend_server = self.getManagedResource("slow_web_server", SlowHTTPServer)
self.addCleanup(backend_server.open)
......@@ -356,8 +279,7 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
def RequestHandler(self):
server = self
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "text/plain")
if self.path == '/set_cookie':
......@@ -378,8 +300,7 @@ class TestBalancer(BalancerTestCase):
"""
__partition_reference__ = 'b'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
# use two backend servers
......@@ -389,16 +310,14 @@ class TestBalancer(BalancerTestCase):
]
return parameter_dict
def test_balancer_round_robin(self):
# type: () -> None
def test_balancer_round_robin(self) -> None:
# requests are by default balanced to both servers
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
{'backend_web_server1', 'backend_web_server2'}
)
def test_balancer_server_down(self):
# type: () -> None
def test_balancer_server_down(self) -> None:
# if one backend is down, it is excluded from balancer
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
......@@ -407,8 +326,7 @@ class TestBalancer(BalancerTestCase):
{'backend_web_server1',}
)
def test_balancer_set_cookie(self):
# type: () -> None
def test_balancer_set_cookie(self) -> None:
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self.assertIn(
......@@ -416,8 +334,7 @@ class TestBalancer(BalancerTestCase):
('default-0', 'default-1'),
)
def test_balancer_respects_sticky_cookie(self):
# type: () -> None
def test_balancer_respects_sticky_cookie(self) -> None:
# if request is made with the sticky cookie, the client stick on one balancer
cookies = dict(SERVERID='default-1')
self.assertEqual(
......@@ -432,8 +349,7 @@ class TestBalancer(BalancerTestCase):
requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
'backend_web_server1')
def test_balancer_stats_socket(self):
# type: () -> None
def test_balancer_stats_socket(self) -> None:
# real time statistics can be obtained by using the stats socket and there
# is a wrapper which makes this a bit easier.
socat_process = subprocess.Popen(
......@@ -458,8 +374,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
"""
__partition_reference__ = 't'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
parameter_dict['dummy_http_server-test-runner-address-list'] = [
......@@ -478,8 +393,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
]
return parameter_dict
def test_use_proper_backend(self):
# type: () -> None
def test_use_proper_backend(self) -> None:
# requests are directed to proper backend based on URL path
test_runner_url_list = self.getRootPartitionConnectionParameterDict(
)['default-test-runner-url-list']
......@@ -532,8 +446,7 @@ class TestHTTP(BalancerTestCase):
"""Check HTTP protocol with a HTTP/1.1 backend
"""
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
# use a HTTP/1.1 server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("HTTP/1.1 Server", EchoHTTP11Server).netloc, 1, False]]
......@@ -541,8 +454,7 @@ class TestHTTP(BalancerTestCase):
__partition_reference__ = 'h'
def test_http_version(self):
# type: () -> None
def test_http_version(self) -> None:
self.assertEqual(
subprocess.check_output([
'curl',
......@@ -558,8 +470,7 @@ class TestHTTP(BalancerTestCase):
b'2',
)
def test_keep_alive(self):
# type: () -> None
def test_keep_alive(self) -> None:
# when doing two requests, connection is established only once
with requests.Session() as session:
session.verify = False
......@@ -594,8 +505,7 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
"""
class RequestHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def do_GET(self):
# type: () -> None
def do_GET(self) -> None:
self.send_response(200)
if self.path == '/':
self.send_header("Content-Length", '0')
......@@ -615,16 +525,14 @@ class TestContentEncoding(BalancerTestCase):
"""
__partition_reference__ = 'ce'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
]
return parameter_dict
def test_gzip_encoding(self):
# type: () -> None
def test_gzip_encoding(self) -> None:
for content_type in (
'text/cache-manifest',
'text/html',
......@@ -652,129 +560,18 @@ class TestContentEncoding(BalancerTestCase):
'{} uses wrong encoding: {}'.format(content_type, resp.headers.get('Content-Encoding')))
self.assertEqual(resp.text, 'OK')
def test_no_gzip_encoding(self):
# type: () -> None
def test_no_gzip_encoding(self) -> None:
resp = requests.get(urllib.parse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
self.assertNotIn('Content-Encoding', resp.headers)
self.assertEqual(resp.text, 'OK')
class CaucaseCertificate(ManagedResource):
"""A certificate signed by a caucase service.
"""
ca_crt_file = None # type: str
crl_file = None # type: str
csr_file = None # type: str
cert_file = None # type: str
key_file = None # type: str
def open(self):
# type: () -> None
self.tmpdir = tempfile.mkdtemp()
self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
self.key_file = os.path.join(self.tmpdir, 'key.pem')
def close(self):
# type: () -> None
shutil.rmtree(self.tmpdir)
@property
def _caucase_path(self):
# type: () -> str
"""path of caucase executable.
"""
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name, caucase):
# type: (str, CaucaseService) -> None
"""Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance.
"""
cas_args = [
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
]
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(self.key_file, 'wb') as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(
NameOID.COMMON_NAME,
common_name,
),
])).sign(
key,
hashes.SHA256(),
default_backend(),
)
with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
csr_id = subprocess.check_output(
cas_args + [
'--send-csr', self.csr_file,
],
).split()[0].decode()
assert csr_id
for _ in range(30):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, self.cert_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as cert_file:
assert 'BEGIN CERTIFICATE' in cert_file.read()
def revoke(self, caucase):
# type: (CaucaseService) -> None
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
'--revoke-crt', self.cert_file, self.key_file,
])
class TestFrontendXForwardedFor(BalancerTestCase):
__partition_reference__ = 'xff'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
certificate = cls.getManagedResource('client_certificate', CaucaseCertificate)
certificate.request('shared frontend', frontend_caucase)
......@@ -791,8 +588,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
parameter_dict['ssl']['frontend-caucase-url-list'] = [frontend_caucase.url]
return parameter_dict
def test_x_forwarded_for_added_when_verified_connection(self):
# type: () -> None
def test_x_forwarded_for_added_when_verified_connection(self) -> None:
client_certificate = self.getManagedResource('client_certificate', CaucaseCertificate)
for backend in ('default', 'default-auth'):
......@@ -805,8 +601,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
).json()
self.assertEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
def test_x_forwarded_for_stripped_when_no_certificate(self):
# type: () -> None
def test_x_forwarded_for_stripped_when_no_certificate(self) -> None:
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
result = requests.get(
balancer_url,
......@@ -822,8 +617,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
verify=False,
)
def test_x_forwarded_for_stripped_when_not_verified_certificate(self):
# type: () -> None
def test_x_forwarded_for_stripped_when_not_verified_certificate(self) -> None:
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
# certificate from an unknown CA
......@@ -855,8 +649,7 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
__partition_reference__ = 's'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
server_caucase = cls.getManagedResource('server_caucase', CaucaseService)
server_certificate = cls.getManagedResource('server_certificate', CaucaseCertificate)
server_certificate.request(cls._ipv4_address, server_caucase)
......@@ -867,8 +660,7 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
parameter_dict['ssl']['key'] = f.read()
return parameter_dict
def test_certificate_validates_with_provided_ca(self):
# type: () -> None
def test_certificate_validates_with_provided_ca(self) -> None:
server_certificate = self.getManagedResource("server_certificate", CaucaseCertificate)
requests.get(self.default_balancer_url, verify=server_certificate.ca_crt_file)
......@@ -877,8 +669,7 @@ class TestClientTLS(BalancerTestCase):
__partition_reference__ = 'c'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
certificate1 = cls.getManagedResource('client_certificate1', CaucaseCertificate)
certificate1.request('client_certificate1', frontend_caucase1)
......@@ -897,8 +688,7 @@ class TestClientTLS(BalancerTestCase):
]
return parameter_dict
def test_refresh_crl(self):
# type: () -> None
def test_refresh_crl(self) -> None:
logger = self.logger
class DebugLogFile:
......@@ -916,8 +706,7 @@ class TestClientTLS(BalancerTestCase):
# when client certificate can be authenticated, backend receive the CN of
# the client certificate in "remote-user" header
def _make_request():
# type: () -> dict
def _make_request() -> dict:
return requests.get(
self.default_balancer_url,
cert=(client_certificate.cert_file, client_certificate.key_file),
......@@ -976,8 +765,7 @@ class TestPathBasedRouting(BalancerTestCase):
__partition_reference__ = 'pbr'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
parameter_dict = super()._getInstanceParameterDict()
parameter_dict['zope-family-dict'][
'second'
......@@ -1003,8 +791,7 @@ class TestPathBasedRouting(BalancerTestCase):
]
return parameter_dict
def test_routing(self):
# type: () -> None
def test_routing(self) -> None:
published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
scheme = 'scheme'
netloc = 'example.com:8080'
......@@ -1015,8 +802,7 @@ class TestPathBasedRouting(BalancerTestCase):
# For easier reading of test data, visually separating the virtual host
# base from the virtual host root
vhr = '/VirtualHostRoot'
def assertRoutingEqual(family, path, expected_path):
# type: (str, str, str) -> None
def assertRoutingEqual(family: str, path: str, expected_path: str) -> None:
# sanity check: unlike the rules, this test is sensitive to outermost
# slashes, and paths must be absolute-ish for code simplicity.
assert path.startswith('/')
......
software/erp5/test/test/test_erp5.py
View file @ b51f5523
......@@ -634,8 +634,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
)):
os.unlink(logfile)
def _getCrontabCommand(self, crontab_name):
# type: (str) -> str
def _getCrontabCommand(self, crontab_name: str) -> str:
"""Read a crontab and return the command that is executed.
overloaded to use crontab from zope partition
......@@ -1046,8 +1045,7 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
__partition_reference__ = 'n'
__test_matrix__ = matrix((neo,))
def _getCrontabCommand(self, crontab_name):
# type: (str) -> str
def _getCrontabCommand(self, crontab_name: str) -> str:
"""Read a crontab and return the command that is executed.
overloaded to use crontab from neo partition
......@@ -1127,7 +1125,7 @@ class TestUnsetWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedU
"""
__partition_reference__ = 'unset-with-max-rlimit-nofile'
def test_unset_with_max_rlimit_nofile(self):
def test_unset_with_max_rlimit_nofile(self) -> None:
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
limit = resource.getrlimit(resource.RLIMIT_NOFILE)
......
software/erp5/test/test/test_mariadb.py
View file @ b51f5523
......@@ -36,6 +36,7 @@ import subprocess
import urllib.parse
import MySQLdb
import MySQLdb.connections
from slapos.testing.utils import CrontabMixin, getPromisePluginParameterDict
......@@ -60,8 +61,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
return "mariadb"
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> dict
def _getInstanceParameterDict(cls) -> dict:
return {
'tcpv4-port': 3306,
'max-connection-count': 5,
......@@ -76,12 +76,10 @@ class MariaDBTestCase(ERP5InstanceTestCase):
}
@classmethod
def getInstanceParameterDict(cls):
# type: () -> dict
def getInstanceParameterDict(cls) -> dict:
return {'_': json.dumps(cls._getInstanceParameterDict())}
def getDatabaseConnection(self):
# type: () -> MySQLdb.connections.Connection
def getDatabaseConnection(self) -> MySQLdb.connections.Connection:
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
......@@ -106,8 +104,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'*/srv/backup/*',
)
def test_full_backup(self):
# type: () -> None
def test_full_backup(self) -> None:
self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
full_backup_file, = glob.glob(
os.path.join(
......@@ -121,8 +118,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
with gzip.open(full_backup_file, 'rt') as dump:
self.assertIn('CREATE TABLE', dump.read())
def test_logrotate_and_slow_query_digest(self):
# type: () -> None
def test_logrotate_and_slow_query_digest(self) -> None:
# slow query digest needs to run after logrotate, since it operates on the rotated
# file, so this tests both logrotate and slow query digest.
......@@ -193,8 +189,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self):
# type: () -> None
def test_utf8_collation(self) -> None:
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
......@@ -219,8 +214,7 @@ class TestMariaDB(MariaDBTestCase):
class TestMroonga(MariaDBTestCase):
def test_mroonga_plugin_loaded(self):
# type: () -> None
def test_mroonga_plugin_loaded(self) -> None:
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("show plugins")
......@@ -229,8 +223,7 @@ class TestMroonga(MariaDBTestCase):
('Mroonga', 'ACTIVE', 'STORAGE ENGINE', 'ha_mroonga.so', 'GPL'),
plugins)
def test_mroonga_normalize_udf(self):
# type: () -> None
def test_mroonga_normalize_udf(self) -> None:
# example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -255,8 +248,7 @@ class TestMroonga(MariaDBTestCase):
self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
# type: () -> None
def test_mroonga_full_text_normalizer(self) -> None:
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -293,8 +285,7 @@ class TestMroonga(MariaDBTestCase):
cnx.store_result().fetch_row(maxrows=2),
)
def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
# type: () -> None
def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self) -> None:
# Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......@@ -337,8 +328,7 @@ class TestMroonga(MariaDBTestCase):
""")
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_stem(self):
# type: () -> None
def test_mroonga_full_text_stem(self) -> None:
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
......
software/seleniumserver/buildout.hash.cfg
View file @ b51f5523
......@@ -19,4 +19,4 @@ md5sum = 10e19df182c692b71ea552da183a0bcf
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = 8c48c7bef34dd54ab3bd94c91f2bf041
md5sum = 5a7abfff9f9d7898620f8c7fc1e6f488
\ No newline at end of file
software/seleniumserver/instance-selenium.cfg.in
View file @ b51f5523
......@@ -25,9 +25,8 @@ fonts =
${liberation-fonts:location}
${ocrb-fonts:location}
$${directory:fonts}
# XXX we don't include conf.d for now, to keep compatibility with current font selection problems.
includes =
# ${fontconfig:location}/etc/fonts/conf.d
${fontconfig:location}/etc/fonts/conf.d
[xvfb-instance]
recipe = slapos.cookbook:wrapper
......@@ -93,26 +92,15 @@ environment =
PATH=${buildout:bin-directory}
XORG_LOCK_DIR=$${directory:tmp}
DISPLAY=$${xvfb-instance:display}
LC_ALL=C.UTF8
FONTCONFIG_FILE=$${fontconfig-conf:output}
hostname = $${slap-configuration:ipv4-random}
[selenium-server-node-instance-firefox-60]
[selenium-server-node-instance-firefox-102]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-60:version},firefox_binary=${firefox-wrapper-60:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.22.0:location}
port = 7777
[selenium-server-node-instance-firefox-68]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-68:version},firefox_binary=${firefox-wrapper-68:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7778
[selenium-server-node-instance-firefox-78]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-78:version},firefox_binary=${firefox-wrapper-78:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-102:version},firefox_binary=${firefox-wrapper-102:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.33.0:location}
port = 7779
[selenium-server-node-instance-firefox-115]
......@@ -121,18 +109,17 @@ capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.33.0:location}
port = 7780
[selenium-server-node-instance-chromium-69]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
port = 7781
[selenium-server-node-instance-chromium-91]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-91:version},chrome_binary=${chromium-wrapper-91:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-91:location}
port = 7782
port = 7781
[selenium-server-node-instance-chromium-120]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-120:version},chrome_binary=${chromium-wrapper-120:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-120:location}
port = 7782
[selenium-server-admin-password]
recipe = slapos.cookbook:generate.password
......@@ -301,12 +288,10 @@ instance-promises =
$${selenium-server-frontend-listen-promise:name}
$${selenium-server-hub-listen-promise:name}
$${selenium-server-hub-nodes-registered-promise:name}
$${selenium-server-node-firefox-60-listen-promise:name}
$${selenium-server-node-firefox-68-listen-promise:name}
$${selenium-server-node-firefox-78-listen-promise:name}
$${selenium-server-node-firefox-102-listen-promise:name}
$${selenium-server-node-firefox-115-listen-promise:name}
$${selenium-server-node-instance-chromium-69-listen-promise:name}
$${selenium-server-node-instance-chromium-91-listen-promise:name}
$${selenium-server-node-instance-chromium-120-listen-promise:name}
[check-port-listening-promise]
......@@ -336,8 +321,8 @@ promise = check_command_execute
name = $${:_buildout_section_name_}.py
config-command =
$${selenium-server-check-nodes-registered:output} $${selenium-server-hub-instance:api-url} $${:expected-node-count}
# We have 6 nodes with 3 slots each
expected-node-count = 18
# We have 4 nodes with 3 slots each
expected-node-count = 12
[selenium-server-check-nodes-registered]
recipe = slapos.recipe.template
......@@ -351,36 +336,25 @@ inline =
sys.exit(0 if expected_node_count == actual_node_count else 1)
[selenium-server-node-firefox-60-listen-promise]
[selenium-server-node-firefox-102-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-60:hostname}
config-port = $${selenium-server-node-instance-firefox-60:port}
[selenium-server-node-firefox-68-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-68:hostname}
config-port = $${selenium-server-node-instance-firefox-68:port}
[selenium-server-node-firefox-78-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-78:hostname}
config-port = $${selenium-server-node-instance-firefox-78:port}
config-host = $${selenium-server-node-instance-firefox-102:hostname}
config-port = $${selenium-server-node-instance-firefox-102:port}
[selenium-server-node-firefox-115-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-115:hostname}
config-port = $${selenium-server-node-instance-firefox-115:port}
[selenium-server-node-instance-chromium-69-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-69:hostname}
config-port = $${selenium-server-node-instance-chromium-69:port}
[selenium-server-node-instance-chromium-91-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-91:hostname}
config-port = $${selenium-server-node-instance-chromium-91:port}
[selenium-server-node-instance-chromium-120-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-120:hostname}
config-port = $${selenium-server-node-instance-chromium-120:port}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
......
software/seleniumserver/software.cfg
View file @ b51f5523
......@@ -64,15 +64,7 @@ install =
dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
os.symlink(extra_font_dir, dst)
[firefox-60]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[firefox-68]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[firefox-78]
[firefox-102]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
......
software/seleniumserver/test/test.py
View file @ b51f5523
......@@ -282,12 +282,12 @@ class TestBrowserSelection(WebServerMixin, SeleniumServerTestCase):
webdriver_url = parameter_dict['backend-url']
desired_capabilities = DesiredCapabilities.FIREFOX.copy()
desired_capabilities['version'] = '60.0.2esr'
desired_capabilities['version'] = '102.15.1esr'
driver = webdriver.Remote(
command_executor=webdriver_url,
desired_capabilities=desired_capabilities)
self.assertIn(
'Gecko/20100101 Firefox/60.0',
'Gecko/20100101 Firefox/102.0',
driver.execute_script('return navigator.userAgent'))
driver.quit()
desired_capabilities['version'] = '115.3.1esr'
......@@ -409,31 +409,18 @@ class TestSSHServer(SeleniumServerTestCase):
self.assertIn(b"Welcome to SlapOS Selenium Server.", received)
class TestFirefox60(
class TestFirefox102(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='60.0.2esr')
user_agent = 'Gecko/20100101 Firefox/60.0'
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='102.15.1esr')
user_agent = 'Gecko/20100101 Firefox/102.0'
class TestFirefox68(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
user_agent = 'Gecko/20100101 Firefox/68.0'
class TestFirefox78(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='78.1.0esr')
user_agent = 'Gecko/20100101 Firefox/78.0'
# resizing window does not work, but we don't really depend on it
@unittest.expectedFailure
def test_resize_window(self):
super().test_resize_window()
class TestFirefox115(
......@@ -450,19 +437,19 @@ class TestFirefox115(
super().test_resize_window()
class TestChrome69(
class TestChrome91(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
user_agent = 'Chrome/69.0.3497.0'
desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
user_agent = 'Chrome/91.0.4472.0'
class TestChrome91(
class TestChrome120(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
user_agent = 'Chrome/91.0.4472.0'
desired_capabilities = dict(DesiredCapabilities.CHROME, version='120.0.6099.109')
user_agent = 'Chrome/120.0.0.0'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7