Commit b62f8386 authored by Jérome Perrin's avatar Jérome Perrin

Merge remote-tracking branch 'origin/master' into zope4py2

parents 25b5b32b b696fd3d
......@@ -45,7 +45,7 @@ class TestBackupServer(InstanceTestCase):
# Check that there is a RSS feed
self.assertTrue('rss' in parameter_dict)
self.assertTrue(parameter_dict['rss'].startswith(
'https://[%s]:9443/' % (self._ipv6_address, )
f'https://[{self._ipv6_address}]:9443/'
))
result = requests.get(
......
......@@ -24,7 +24,6 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import requests
......
......@@ -51,7 +51,6 @@ setup(name=name,
# caucase needed to connect to the KeDiFa caucase
'caucase',
'cryptography',
'backports.lzma',
],
zip_safe=True,
test_suite='test',
......
......@@ -51,18 +51,13 @@ import urllib.parse
import socket
import sys
import logging
import lzma
import random
import string
from slapos.slap.standalone import SlapOSNodeInstanceError
import caucase.client
import caucase.utils
try:
import lzma
except ImportError:
from backports import lzma
import datetime
from cryptography import x509
......
......@@ -45,7 +45,6 @@ setup(name=name,
'slapos.cookbook',
'slapos.libnetworkcache',
'requests',
'six',
'PyPDF2',
],
zip_safe=True,
......
##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
......@@ -31,8 +30,8 @@ import csv
import multiprocessing
import os
import json
import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import xmlrpc.client as xmlrpclib
import urllib.parse as urllib_parse
import ssl
import base64
import io
......@@ -63,6 +62,7 @@ class CloudOooTestCase(_CloudOooTestCase):
context=ssl_context,
allow_none=True,
)
self.addCleanup(self.server('close'))
def normalizeFontName(font_name):
......@@ -111,16 +111,16 @@ class HTMLtoPDFConversionFontTestMixin:
def test(self):
actual_font_mapping_mapping = {}
for font in self.expected_font_mapping:
src_html = '''
src_html = f'''
<style>
p {{ font-family: "{font}"; font-size: 20pt; }}
</style>
<p>the quick brown fox jumps over the lazy dog.</p>
<p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
'''.format(**locals())
'''
pdf_data = self._convert_html_to_pdf(src_html)
pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
pdf_reader = PyPDF2.PdfFileReader(io.BytesIO(pdf_data))
self.assertEqual(
self.pdf_producer,
pdf_reader.getDocumentInfo()['/Producer'])
......@@ -165,7 +165,7 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LiberationSans',
'OpenSymbol': set(['DejaVuSans', 'OpenSymbol']),
'OpenSymbol': {'DejaVuSans', 'OpenSymbol'},
'Palatino': 'LiberationSans',
'Roboto Black': 'LiberationSans',
'Roboto Condensed Light': 'LiberationSans',
......@@ -180,9 +180,9 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
return base64.decodebytes(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
base64.encodebytes(src_html.encode()).decode(),
'html',
'pdf',
False,
......@@ -238,9 +238,9 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
return base64.decodebytes(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
base64.encodebytes(src_html.encode()).decode(),
'html',
'pdf',
).encode())
......@@ -251,10 +251,10 @@ class TestLibreOfficeTextConversion(CloudOooTestCase):
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
base64.decodebytes(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
base64.encodebytes(
'<html>héhé</html>'.encode()).decode(),
'html',
'txt',
).encode()),
......@@ -274,19 +274,18 @@ class TestLibreOfficeCluster(CloudOooTestCase):
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
return base64.decodebytes(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
base64.encodebytes(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
with pool:
converted = pool.map(
_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
......@@ -294,9 +293,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
reader = csv.DictReader(io.StringIO(res.text))
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
......@@ -309,8 +307,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# process at least one request.
self.assertGreater(total_hrsp_2xx[backend], 0)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
......
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
......
......@@ -24,7 +24,6 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import absolute_import
from setuptools import setup, find_packages
version = '0.0.1.dev0'
......@@ -49,7 +48,6 @@ setup(name=name,
'psutil',
'requests',
'mysqlclient',
'backports.lzma',
'cryptography',
'pexpect',
'pyOpenSSL',
......
......@@ -25,7 +25,6 @@
#
##############################################################################
from __future__ import absolute_import
import json
import os
......
This diff is collapsed.
This diff is collapsed.
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
......@@ -26,11 +25,10 @@
#
##############################################################################
from __future__ import absolute_import
import os
import json
import glob
import six.moves.urllib.parse
import urllib.parse
import socket
import sys
import time
......@@ -39,7 +37,7 @@ import datetime
import subprocess
import gzip
from backports import lzma
import lzma
import MySQLdb
from slapos.testing.utils import CrontabMixin
......@@ -84,7 +82,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
# type: () -> MySQLdb.connections.Connection
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
......@@ -208,7 +206,7 @@ class TestMariaDB(MariaDBTestCase):
"""
select * from test_utf8_collation where col1 = "a"
""")
self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
class TestMroonga(MariaDBTestCase):
......@@ -232,7 +230,7 @@ class TestMroonga(MariaDBTestCase):
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
# XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
......@@ -245,7 +243,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
......@@ -282,7 +280,7 @@ class TestMroonga(MariaDBTestCase):
WHERE MATCH (content) AGAINST ("+ブラック" IN BOOLEAN MODE)
""")
self.assertEqual(
((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
cnx.store_result().fetch_row(maxrows=2),
)
......
......@@ -16,7 +16,6 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import absolute_import
import json
import os.path
import unittest
......@@ -76,5 +75,5 @@ def lookupMount(zurl):
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
with open(path) as f:
return f.read()
......@@ -44,7 +44,6 @@ setup(name=name,
'slapos.core',
'supervisor',
'slapos.libnetworkcache',
'typing; python_version<"3"',
],
test_suite='test',
)
......@@ -35,8 +35,8 @@ import tempfile
import time
import requests
import six.moves.urllib as urllib
import six.moves.xmlrpc_client
import urllib.parse
import xmlrpc.client
import urllib3
from slapos.grid.utils import md5digest
......@@ -83,8 +83,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
@classmethod
def setUpClass(cls):
# request and instanciate with old software url
super(ERP5UpgradeTestCase, cls).setUpClass()
# request and instantiate with old software url
super().setUpClass()
cls.setUpOldInstance()
......@@ -155,7 +155,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
# wait for old site creation
cls.session.get(
'{zope_base_url}/person_module'.format(zope_base_url=cls.zope_base_url),
f'{cls.zope_base_url}/person_module',
auth=requests.auth.HTTPBasicAuth(
username=param_dict['inituser-login'],
password=param_dict['inituser-password'],
......@@ -171,14 +171,10 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
erp5_xmlrpc_client = xmlrpc.client.ServerProxy(
cls.authenticated_zope_base_url,
context=ssl_context,
)
# BBB use as a context manager only on python3
if sys.version_info < (3, ):
yield erp5_xmlrpc_client
else:
with erp5_xmlrpc_client:
yield erp5_xmlrpc_client
......@@ -188,7 +184,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
try:
custom.manage_addProduct.PythonScripts.manage_addPythonScript(
script_id)
except six.moves.xmlrpc_client.ProtocolError as e:
except xmlrpc.client.ProtocolError as e:
if e.errcode != 302:
raise
getattr(custom, script_id).ZPythonScriptHTML_editAction(
......
......@@ -18,4 +18,4 @@ md5sum = 8357771b70efd0740561b1cb46f6955e
[template-deploy-test]
filename = instance.cfg.in
md5sum = c5db797980951b764b69aaa4b60a0380
md5sum = 062e677990ca0cd0c0396993c58b46a2
......@@ -22,7 +22,7 @@ parts =
[request-kvm]
<= slap-connection
recipe = slapos.cookbook:request
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
software-type = kvm
name = test-kvm
......
......@@ -25,24 +25,22 @@ command =
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
template = inline:
output = $${buildout:directory}/bin/$${:_buildout_section_name_}
inline =
#!/bin/sh
export PATH=${python2.7-with-eggs:location}:$PATH
exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory', 'playbook/roles/standalone-shared') }}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
mode = 0755
context =
key slapparameter_dict slap-configuration:configuration
key partition_ipv4 slap-configuration:ipv4
[switch_softwaretype]
default = $${:deploy-test}
deploy-test = dynamic-template-deploy-test:rendered
deploy-test = dynamic-template-deploy-test:output
[dynamic-template-deploy-test]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
rendered = $${buildout:directory}/template-deploy-test.cfg
url = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
output = $${buildout:directory}/template-deploy-test.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
......@@ -52,4 +50,3 @@ context =
raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
raw waittime ${deploy-script-controller-script:waittime}
raw tries ${deploy-script-controller-script:tries}
mode = 0644
[buildout]
extends =
buildout.hash.cfg
https://lab.nexedi.com/nexedi/slapos/raw/1.0.264/software/kvm/software.cfg
https://lab.nexedi.com/nexedi/slapos/raw/1.0.271/software/kvm/software.cfg
parts =
python-with-eggs
template-deploy-test
......@@ -15,23 +15,8 @@ eggs =
requests
interpreter = ${:_buildout_section_name_}
[python2.7-with-eggs]
# create interpreters named "python" and "python2.7" so that
# instance profile can use them in $PATH
recipe = plone.recipe.command
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
ln -s ${buildout:bin-directory}/${python-with-eggs:interpreter} ${:location}/python &&
ln -s ${buildout:bin-directory}/${python-with-eggs:interpreter} ${:location}/python2.7
update-command = ${:command}
[playbook]
recipe = plone.recipe.command
stop-on-error = true
environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
location = ${buildout:parts-directory}/${:_buildout_section_name_}
......@@ -70,7 +55,6 @@ location = ${:_profile_base_location_}/${:filename}
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
erp5.util = 0.4.69
[buildout]
extends =
software.cfg
[python]
part = python2.7
......@@ -46,7 +46,6 @@ setup(name=name,
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'six',
],
zip_safe=True,
test_suite='test',
......
......@@ -34,11 +34,10 @@ import struct
import subprocess
import tempfile
import time
import six
import sys
from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
from six.moves.socketserver import StreamRequestHandler, TCPServer
from http.server import SimpleHTTPRequestHandler
from socketserver import StreamRequestHandler, TCPServer
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
......@@ -49,8 +48,8 @@ FLUSH_INTERVAL = 1
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py2" if six.PY2 else ""))))
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class FluentdTestCase(SlapOSInstanceTestCase):
__partition_reference__ = 'fluentd'
......@@ -135,11 +134,11 @@ class WendelinTutorialTestCase(FluentdTestCase):
return subprocess.check_output(
[self._fluentd_bin, '-c', conf_path, '--dry-run'],
env={'GEM_PATH': self._gem_path},
universal_newlines=True,
text=True,
)
def _test_configuration(self, expected_str):
self.assertRegexpMatches(
self.assertRegex(
self.read_fluentd_conf(self._conf),
expected_str,
)
......@@ -168,12 +167,12 @@ class SensorConfTestCase(WendelinTutorialTestCase):
@classmethod
def sensor_conf(cls, script_path):
return '''\
return f'''\
<source>
@type exec
tag tag.name
command %s %s
run_interval %ss
command {sys.executable} {script_path}
run_interval {FLUSH_INTERVAL}s
<parse>
keys pressure, humidity, temperature
</parse>
......@@ -182,25 +181,25 @@ class SensorConfTestCase(WendelinTutorialTestCase):
@type forward
<server>
name myserver1
host %s
host {cls._ipv6_address}
</server>
<buffer>
flush_mode immediate
</buffer>
</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
</match>'''
@classmethod
def sensor_script(cls, measurementList):
return '''\
#!/usr/bin/python
measurement_text = "\t".join(measurementList)
return f'''\
#!{sys.executable}
# -*- coding: utf-8 -*-
print("%s")''' % "\t".join(measurementList)
print("{measurement_text}")'''
def test_configuration(self):
self._test_configuration(
r'adding forwarding server \'myserver1\' host="%s" port=%s weight=60'
% (self._ipv6_address, FLUENTD_PORT)
fr'adding forwarding server \'myserver1\' host="{self._ipv6_address}" port={FLUENTD_PORT} weight=60'
)
def test_send_data(self):
......@@ -229,25 +228,24 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
@classmethod
def gateway_conf(cls, fluentd_port, wendelin_port):
return '''\
return f'''\
<source>
@type forward
port %s
bind %s
port {fluentd_port}
bind {cls._ipv6_address}
</source>
<match tag.name>
@type wendelin
streamtool_uri http://[%s]:%s/erp5/portal_ingestion_policies/default
streamtool_uri http://[{cls._ipv6_address}]:{wendelin_port}/erp5/portal_ingestion_policies/default
user foo
password bar
<buffer>
flush_mode interval
@type file
path fluentd-buffer-file/
flush_interval %ss
flush_interval {FLUSH_INTERVAL}s
</buffer>
</match>''' % (fluentd_port, cls._ipv6_address, cls._ipv6_address,
wendelin_port, FLUSH_INTERVAL)
</match>'''
@classmethod
def get_configuration(cls):
......
......@@ -27,7 +27,7 @@
import os
import logging
from six.moves.urllib.parse import urlparse
from urllib.parse import urlparse
import requests
......
......@@ -48,7 +48,6 @@ setup(name=name,
'erp5.util',
'supervisor',
'psutil',
'six',
],
zip_safe=True,
test_suite='test',
......
This diff is collapsed.
......@@ -45,7 +45,6 @@ setup(name=name,
'slapos.cookbook',
'slapos.libnetworkcache',
'supervisor',
'six',
'requests'
],
zip_safe=True,
......
##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
......@@ -28,7 +27,7 @@
import os
import json
from six.moves.urllib import parse
from urllib import parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......
......@@ -28,11 +28,11 @@
import os
import json
import glob
import urlparse
import urllib.parse
import socket
import time
import re
import BaseHTTPServer
import http.server
import multiprocessing
import subprocess
......@@ -44,7 +44,7 @@ from . import setUpModule
setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
class TestPublishedURLIsReachableMixin:
"""Mixin that checks that default page of ERP5 is reachable.
"""
......@@ -52,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
virtual_host_url = urllib.parse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
......@@ -72,7 +72,7 @@ class TestPublishedURLIsReachableMixin(object):
total=60,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
with session:
r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
......@@ -80,7 +80,7 @@ class TestPublishedURLIsReachableMixin(object):
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
urllib.parse.urljoin(base_url, f'{site_id}/login_form'),
verify=verify,
allow_redirects=False,
)
......@@ -134,7 +134,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
f'https://[{self._ipv6_address}]:8888/tree',
param_dict['jupyter-url']
)
......@@ -172,7 +172,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
}
def checkValidHTTPSURL(self, url):
parsed = urlparse.urlparse(url)
parsed = urllib.parse.urlparse(url)
self.assertEqual(parsed.scheme, 'https')
self.assertTrue(parsed.hostname)
self.assertTrue(parsed.port)
......@@ -182,16 +182,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)])
param_dict[f'family-{family_name}'])
self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format(family_name=family_name)])
param_dict[f'family-{family_name}-v6'])
def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per family
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[
'{family_name}-test-runner-url-list'.format(family_name=family_name)]
f'{family_name}-test-runner-url-list']
self.assertEqual(3, len(family_test_runner_url_list))
for url in family_test_runner_url_list:
self.checkValidHTTPSURL(url)
......@@ -209,23 +209,23 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
# normal access on ipv4 and ipv6 and test runner access on ipv4 only
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
process_info, = (p for p in all_process_info if p['name'] == 'apache')
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
sorted([
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
]))
))
def test_haproxy_listen(self):
# There is one haproxy per family
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [
process_info, = (
p for p in all_process_info if p['name'].startswith('haproxy-')
]
)
haproxy_process = psutil.Process(process_info['pid'])
self.assertEqual([socket.AF_INET, socket.AF_INET], [
c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
......@@ -290,8 +290,8 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
zodb["pool-timeout"] = "10m"
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
with open(f'{partition}/etc/zope-{zope}.conf') as f:
conf = list(map(str.strip, f.readlines()))
i = conf.index("<zodb_db root>") + 1
conf = iter(conf[i:conf.index("</zodb_db>", i)])
for line in conf:
......@@ -300,23 +300,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
if line == '</zeoclient>':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
for k, v in storage.items():
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
for k, v in zodb.items():
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
for zope in range(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
for zope in range(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
......@@ -332,19 +332,20 @@ def popenCommunicate(command_list, input_=None, **kwargs):
popen.kill()
if popen.returncode != 0:
raise ValueError(
'Issue during calling %r, result was:\n%s' % (command_list, result))
f'Issue during calling {command_list!r}, result was:\n{result}')
return result
class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class TestHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
response = {
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
}
response = json.dumps(response, indent=2)
'Incoming Headers': {k.lower(): v for k, v in self.headers.items()},
},
indent=2,
).encode('utf-8')
self.end_headers()
self.wfile.write(response)
......@@ -352,7 +353,7 @@ class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
"""This check deployment script like instantiation
Low level assertions are done here in roder to assure that
Low level assertions are done here in order to assure that
https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
slapos-master-standalone.yml
works correctly
......@@ -426,8 +427,8 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
1,
len(backend_apache_configuration_list)
)
backend_apache_configuration = open(
backend_apache_configuration_list[0]).read()
with open(backend_apache_configuration_list[0]) as f:
backend_apache_configuration = f.read()
self.assertIn(
'SSLVerifyClient require',
backend_apache_configuration
......@@ -452,7 +453,7 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
common_name = 'TEST-SSL-AUTH'
popenCommunicate([
'openssl', 'req', '-utf8', '-nodes', '-config', openssl_config, '-new',
'-keyout', key, '-out', csr, '-days', '3650'], '%s\n' % (common_name,),
'-keyout', key, '-out', csr, '-days', '3650'], f'{common_name}\n'.encode(),
stdin=subprocess.PIPE)
popenCommunicate([
'openssl', 'ca', '-utf8', '-days', '3650', '-batch', '-config',
......@@ -464,11 +465,14 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
ip, port = re.search(
r'.*http:\/\/(.*):(\d*)\/.*', portal_slap_line).groups()
port = int(port)
server = BaseHTTPServer.HTTPServer((ip, port), TestHandler)
server = http.server.HTTPServer((ip, port), TestHandler)
server_process = multiprocessing.Process(
target=server.serve_forever, name='HTTPServer')
server_process.start()
try:
self.addCleanup(server_process.terminate)
self.addCleanup(server_process.join, 10)
server.socket.close()
# assert that accessing the service endpoint results with certificate
# authentication and proper information extraction
result_json = requests.get(
......@@ -482,6 +486,3 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
'/erp5/portal_slap/',
result_json['Path']
)
finally:
server_process.join(10)
server_process.terminate()
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
......@@ -29,7 +28,7 @@
import os
import json
import glob
import urlparse
import urllib.parse
import socket
import sys
import time
......@@ -38,7 +37,7 @@ import datetime
import subprocess
import gzip
from backports import lzma
import lzma
import MySQLdb
from slapos.testing.utils import CrontabMixin
......@@ -80,7 +79,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
def getDatabaseConnection(self):
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
......@@ -91,6 +90,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
host=db_url.hostname,
port=db_url.port,
db=database_name,
use_unicode=True,
charset='utf8mb4'
)
......@@ -106,7 +107,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'mariadb-full',
'20500101000000.sql.gz',
),
'r') as dump:
'rt') as dump:
self.assertIn('CREATE TABLE', dump.read())
def test_logrotate_and_slow_query_digest(self):
......@@ -148,7 +149,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'slowquery_digest',
'slowquery_digest.txt-2050-01-01.xz',
)
with lzma.open(slow_query_report, 'r') as f:
with lzma.open(slow_query_report, 'rt') as f:
# this is the hash for our "select sleep(n)" slow query
self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
......@@ -170,7 +171,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
self.assertEqual(
error_context.exception.output,
"""\
b"""\
Threshold is lower than expected:
Expected total queries : 1.0 and current is: 2
Expected slowest query : 0.1 and current is: 3
......@@ -220,7 +221,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
self.assertEqual((('abcdあぃうぇリットル',),),
self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
......@@ -233,7 +234,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual((('ABCDあぃうぇ㍑',),),
self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
......@@ -321,7 +322,7 @@ class TestMroonga(MariaDBTestCase):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT mroonga_command('register token_filters/stem')")
self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
CREATE TABLE memos (
......
......@@ -75,5 +75,5 @@ def lookupMount(zurl):
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
with open(path) as f:
return f.read()
......@@ -17,7 +17,6 @@ changes to the code, run tests and publish changes.
```bash
# install this software release and request an instance
# use software-py3.cfg instead of software.cfg if the SR you want to test is written in Python 3
SR=https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
COMP=slaprunner
INSTANCE_NAME=$COMP
......
[buildout]
extends =
software.cfg
[python]
part = python3
[python-interpreter]
extra-eggs +=
# plantuml 0.3.0 is only available for Python 3
${slapos.test.plantuml-setup:egg}
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
caucase ${slapos.test.caucase-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
galene ${slapos.test.galene-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
hugo ${slapos.test.hugo-setup:setup}
jscrawler ${slapos.test.jscrawler-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
matomo ${slapos.test.matomo-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
theia ${slapos.test.theia-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
......@@ -27,9 +27,6 @@ parts =
shared-part-list =
[python]
part = python2.7
[setup-develop-egg]
recipe = zc.recipe.egg:develop
......@@ -250,8 +247,7 @@ egg = slapos.core
setup = ${slapos.core-repository:location}
[python-interpreter]
eggs += ${:extra-eggs}
extra-eggs =
eggs +=
${lxml-python:egg}
${python-PyYAML:egg}
${slapos.core-setup:egg}
......@@ -289,10 +285,11 @@ extra-eggs =
${slapos.test.kvm-setup:egg}
${slapos.test.matomo-setup:egg}
${slapos.test.metabase-setup:egg}
${slapos.test.ors-amarisoft-setup:egg}
${slapos.test.monitor-setup:egg}
${slapos.test.nextcloud-setup:egg}
${slapos.test.nginx-push-stream-setup:egg}
${slapos.test.ors-amarisoft-setup:egg}
${slapos.test.plantuml-setup:egg}
${slapos.test.powerdns-setup:egg}
${slapos.test.proftpd-setup:egg}
${slapos.test.re6stnet-setup:egg}
......@@ -352,27 +349,46 @@ context =
tests =
json-schemas ${slapos.cookbook-setup:setup}
# The following list is for SR that work with either Python 2 and 3
# (as main Python). The test egg must supply a URL which depends on
# the version of Python that is used to run the test.
# Due to a bug in the way promises are run, we may also list some Py3-only SR
# here, to check there's no promise issue when slapos node runs with Python 2.
erp5 ${slapos.test.erp5-setup:setup}
fluentd ${slapos.test.fluentd-setup:setup}
###
${:extra}
extra =
# WARNING: This is for SR that only support Python 2.
# You should not add more lines here.
backupserver ${slapos.test.backupserver-setup:setup}
beremiz-ide ${slapos.test.beremiz-ide-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
caucase ${slapos.test.caucase-setup:setup}
cloudooo ${slapos.test.cloudooo-setup:setup}
dream ${slapos.test.dream-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
fluentd ${slapos.test.fluentd-setup:setup}
galene ${slapos.test.galene-setup:setup}
gitlab ${slapos.test.gitlab-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
hugo ${slapos.test.hugo-setup:setup}
jscrawler ${slapos.test.jscrawler-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
matomo ${slapos.test.matomo-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
re6stnet ${slapos.test.re6stnet-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
[versions]
......@@ -397,11 +413,9 @@ PyPDF2 = 1.26.0+SlapOSPatched001
# Django 1.11 is python 2 compatible
Django = 1.11
mock = 2.0.0:whl
testfixtures = 6.11.0
funcsigs = 1.0.2
mysqlclient = 1.3.12
pexpect = 4.8.0
ptyprocess = 0.6.0
typing = 3.7.4.3
psycopg2 = 2.8.6
......@@ -48,7 +48,6 @@ setup(name=name,
'supervisor',
'psutil',
'paramiko',
'six',
'requests',
],
zip_safe=True,
......
......@@ -35,12 +35,11 @@ import subprocess
import json
import time
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import quote
from six.moves.urllib.parse import urljoin
from six.moves.configparser import ConfigParser
from urllib.parse import urlparse
from urllib.parse import quote
from urllib.parse import urljoin
from configparser import ConfigParser
import requests
import six
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -191,7 +190,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
takeover_password = parameter_dict["takeover-%s-password" % scope]
resp = requests.get(
"%s?password=%s" % (takeover_url, takeover_password),
f"{takeover_url}?password={takeover_password}",
verify=True)
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertNotIn("Error", resp.text,
......@@ -363,7 +362,7 @@ class TestSSH(SlaprunnerTestCase):
self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), fingerprint_from_url)
fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
class KeyPolicy(object):
class KeyPolicy:
"""Accept server key and keep it in self.key for inspection
"""
def missing_host_key(self, client, hostname, key):
......@@ -507,7 +506,7 @@ class TestResilientInstance(SlaprunnerTestCase):
# just check that keys returned on requested partition are for resilient
self.assertSetEqual(
set(self.computer_partition.getConnectionParameterDict().keys()),
set([
{
'backend-url',
'feed-url-runner-1-pull',
'feed-url-runner-1-push',
......@@ -520,7 +519,7 @@ class TestResilientInstance(SlaprunnerTestCase):
'takeover-runner-1-password',
'takeover-runner-1-url',
'url',
'webdav-url']))
'webdav-url'})
class TestResilientCustomFrontend(TestCustomFrontend):
instance_max_retry = 20
......@@ -601,5 +600,5 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
self.assertTrue(result_after.startswith("Hello"), result_after)
self.assertIn(result, result_after,
"%s not in %s" % (result, result_after))
f"{result} not in {result_after}")
......@@ -391,7 +391,9 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
try:
self.addCleanup(theia_shell_process.wait)
self.addCleanup(theia_shell_process.terminate)
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
# Launch slapos node software from theia shell
......@@ -433,11 +435,6 @@ class TestTheiaEnv(TheiaTestCase):
self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
finally:
# Cleanup the theia shell process
theia_shell_process.terminate()
theia_shell_process.wait()
class ResilientTheiaMixin(object):
@classmethod
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment