diff --git a/component/bzip2/buildout.cfg b/component/bzip2/buildout.cfg index 7633a9efa58213f05c844c31e983bf415506731c..a27ecb5a258899ae1db8980aed4e3916c940d7d4 100644 --- a/component/bzip2/buildout.cfg +++ b/component/bzip2/buildout.cfg @@ -4,7 +4,7 @@ parts = [bzip2] recipe = slapos.recipe.cmmi -url = http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz +url = https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz md5sum = 00b516f4704d4a7cb50a1d97e6e8e15b configure-command = true make-options = diff --git a/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg b/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg new file mode 100644 index 0000000000000000000000000000000000000000..bbad91df54ae8bb077bc3378fc605c7ce97b387c --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg @@ -0,0 +1,35 @@ +# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax. +# The only allowed lines here are (regexes): +# - "^#" comments, copied verbatim +# - "^[" section beginings, copied verbatim +# - lines containing an "=" sign which must fit in the following categorie. +# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file +# But avoid directories, they are not portable. +# Copied verbatim. +# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported +# by the re-generation script. +# Re-generated. +# - other lines are copied verbatim +# Substitution (${...:...}), extension ([buildout] extends = ...) and +# section inheritance (< = ...) are NOT supported (but you should really +# not need these here). + +[instance-profile] +filename = instance.cfg.in +md5sum = 45c4adaae557bed5c25c24f4c4506b3d + +[template-fluentd-config] +filename = templates/fluentd-conf.cnf.in +md5sum = 46eb9943800fe5909953042cef596b6c + +[template-caddy-service] +filename = templates/template-caddy-service.sh.in +md5sum = 3e09969c479931ca4df6546abbd1a117 + +[template-caddyfile] +filename = templates/Caddyfile.in +md5sum = 8644e98038ab76a8e5a499e22f92660a + +[template-caddy-fluentd] +filename = instance-caddy-fluentd.cfg.in +md5sum = a022edfb2b08e8376b7f4140f8d87917 diff --git a/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in b/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..77342aac9e4ab63ac987baf38f51430d45f627b5 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in @@ -0,0 +1,109 @@ +[buildout] +parts = + slapos-test-runner + directory + fluentd-config + fluentd-service + caddy-service + publish-connection-information + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[directory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +bin = $${buildout:directory}/bin +srv = $${buildout:directory}/srv +var = $${buildout:directory}/var +run = $${:var}/run +log = $${:var}/log +scripts = $${:etc}/run +services = $${:etc}/service +promise = $${:etc}/promise/ +www = $${:srv}/www +home = $${:etc}/home +ssl = $${:etc}/ssl +working-dir = $${buildout:directory}/tmp/ + +################################# +# fluentd service +################################# + +[fluentd-config] +recipe = slapos.recipe.template:jinja2 +template = ${template-fluentd-config:output} +rendered = $${directory:etc}/fluentd-conf.cnf +mode = 0600 +context = + key local_ip caddy-configuration:local_ip + +[fluentd-service] +recipe = slapos.cookbook:wrapper +wrapper-path = $${directory:services}/fluentd-service +path = ${fluentd:location} +command-line = ${fluentd:location}/bin/fluentd -vv + -c $${directory:etc}/fluentd-conf.cnf -p ${fluentd-plugin-dev-repository:location}/lib/fluent/plugin/ -o $${directory:etc}/fluent.log +environment = + GEM_PATH= ${fluentd:location}/lib/ruby/gems/1.8/ +output = $${:wrapper-path} + +################################# +# caddy service +################################# + +[caddy-service] +recipe = slapos.recipe.template:jinja2 +template = ${template-caddy-service:output} +rendered = $${directory:services}/caddy +mode = 0700 +context = + key caddy_exec caddy-exec-dict:caddy-exec-file + key caddy_file caddy-configuration:rendered + key pidfile_dir directory:etc + +[caddy-exec-dict] +caddy-exec-file = ${gowork:bin}/caddy + +[caddy-configuration] +recipe = slapos.recipe.template:jinja2 +template = ${template-caddyfile:output} +rendered = $${directory:etc}/Caddyfile +mode = 0600 +access_log = $${directory:log}/caddy-access.log +error_log = $${directory:log}/caddy-error.log +local_ip = $${slap-network-information:local-ipv4} +context = + key local_ip caddy-configuration:local_ip + +[publish-connection-information] +recipe = slapos.cookbook:publish +url = http://$${caddy-configuration:local_ip}:4443 + +[download-source] +recipe = slapos.recipe.build:gitclone +git-executable = ${git:location}/bin/git + +[slapos] +<= download-source +repository = ${slapos-repository:location} + +[create-directory] +recipe = slapos.cookbook:mkdirectory +bin = $${buildout:directory}/bin + +[slapos-test-runner] +recipe = slapos.cookbook:wrapper +wrapper-path = $${directory:bin}/runTestSuite +command-line = + ${buildout:bin-directory}/runTestSuite + --python_interpreter=${buildout:bin-directory}/${eggs:interpreter} + --source_code_path_list=$${slapos:location}/software/erp5testnode/testsuite/fluentTest/tests/ +environment = + PATH=${buildout:bin-directory}:/usr/bin/:/bin/ + LOCAL_IPV4=$${slap-network-information:local-ipv4} + GLOBAL_IPV6=$${slap-network-information:global-ipv6} + FLUENT_SERVICE = $${fluentd-service:path} + SLAPOS_TEST_WORKING_DIR=$${directory:working-dir} + CADDY_DIR = $${directory:etc}/caddy_pidfile \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/instance.cfg.in b/software/erp5testnode/testsuite/fluentTest/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..b31974dfeb6a135af2a1f80a060f01a91cfb16a7 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/instance.cfg.in @@ -0,0 +1,51 @@ +############################# +# +# Deploy caddy instance +# +############################# +[buildout] +parts = + switch-softwaretype + +# publish-connection-parameter + +# Define egg directories to be the one from Software Release +# (/opt/slapgrid/...) +# Always the same. +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[switch-softwaretype] +recipe = slapos.cookbook:softwaretype +default = $${:caddy} +caddy = $${dynamic-template-caddy-fluentd:rendered} + +[dynamic-template-caddy-fluentd] +recipe = slapos.recipe.template:jinja2 +template = ${template-caddy-fluentd:output} +rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename} +filename = instance-caddy-fluentd.cfg + +[slap-connection] +computer-id = $${slap_connection:computer_id} +partition-id = $${slap_connection:partition_id} +server-url = $${slap_connection:server_url} +software-release-url = $${slap_connection:software_release_url} +key-file = $${slap_connection:key_file} +cert-file = $${slap_connection:cert_file} + +[instance-parameter] +# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance. +# We use the slapconfiguration recipe with a few parameters (partition id, +# computer id, certificate, etc). +# It will then authenticate to SlapOS Master and fetch the instance parameters. +# The parameters are accessible from {instance-parameter:configuration.name-of-parameter} +# Always the same. Just copy/paste. +# See docstring of slapos.cookbook:slapconfiguration for more information. +recipe = slapos.cookbook:slapconfiguration +computer = $${slap_connection:computer_id} +partition = $${slap_connection:partition_id} +url = $${slap_connection:server_url} +key = $${slap_connection:key_file} +cert = $${slap_connection:cert_file} \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/software.cfg b/software/erp5testnode/testsuite/fluentTest/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..a66a12d3b0980f2562385d6c0735778a01e13dd1 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/software.cfg @@ -0,0 +1,125 @@ +[buildout] + +extends = + ../../../../stack/slapos.cfg + ../../../../software/caddy/software.cfg + ../../../../software/fluentd/software.cfg + buildout.hash.cfg + +index = https://pypi.python.org/simple/ + +parts = + slapos-cookbook + eggs + caddy + fluentd + fluentd-plugin-dev-repository + instance-profile + template-fluentd-config + +[setup-develop-egg] +recipe = zc.recipe.egg:develop + +[slapos.test.fluentTest-setup] +<= setup-develop-egg +egg = slapos.test.fluentTest +setup = ${slapos-repository:location}/software/erp5testnode/testsuite/fluentTest/tests/ + +[erp5.util-setup] +<= setup-develop-egg +egg = erp5.util[testnode] +setup = ${erp5.util-repository:location} + +[eggs] +recipe = zc.recipe.egg +eggs = + ${slapos.test.fluentTest-setup:egg} + ${erp5.util-setup:egg} + slapos.core +entry-points = + runTestSuite=erp5.util.testsuite:runTestSuite +scripts = + runTestSuite + slapos +interpreter= + python_for_test + +[git-clone-repository] +recipe = slapos.recipe.build:gitclone +git-executable = ${git:location}/bin/git +forbid-download-cache = true +branch = master + +[slapos-repository] +<= git-clone-repository +#repository = https://lab.nexedi.com/nexedi/slapos.git +repository = https://lab.nexedi.com/Sokhoyan/slapos.git +branch = FluentTest_withFixUtil + +# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite +# later we can stop fetching it from git and just use egg +[erp5.util-repository] +<= git-clone-repository +repository = https://lab.nexedi.com/nexedi/erp5.git +revision = 3b9d23bff23cfba208131b004ae4a0e7484e9aff + +[versions] +# clear the version of tested eggs, to make sure we installed the developped ones +slapos.test.fluentTest = +erp5.util = +#erp5.util = 0.4.51 + +[instance-profile] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/instance.cfg +mode = 0644 + +[template-fluentd-config] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/fluentd-conf.cnf.in +mode = 0644 + +[fluentd-plugin-dev-repository] +recipe = slapos.recipe.build:gitclone +repository = https://lab.nexedi.com/nexedi/fluent-plugin-wendelin.git +# dir is pretty name as top-level -dev recipe +git-executable = ${git:location}/bin/git + +[fluentd] +gems = + fluentd==0.14.14 + fluent-plugin-td + bundler + fluent-plugin-bin + +[template-caddy-service] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/template-caddy-service.sh.in +mode = 0644 + +[template-caddyfile] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/Caddyfile.in +mode = 0644 + +[template-caddy-fluentd] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/instance-caddy-fluentd.cfg.in +mode = 0644 + +[caddy] +recipe = slapos.recipe.cmmi +path = ${go_github.com_mholt_caddy:location} +go = ${gowork:golang}/bin/go +configure-command = : +make-targets = +make-binary = cd ${:path}/caddy && ${:go} install -v +environment = + PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s + GOPATH=${gowork:directory} +output = ${gowork:bin}/caddy \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in b/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in new file mode 100644 index 0000000000000000000000000000000000000000..a885d76120719a23e33ee5d696881962979d66de --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in @@ -0,0 +1,4 @@ +{{local_ip}}:4443 + + proxy / {{local_ip}}:9443 + diff --git a/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in b/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in new file mode 100644 index 0000000000000000000000000000000000000000..bec70555184466e76099c3904978fc063fde3f33 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in @@ -0,0 +1,55 @@ + + @type forward + port 5438 + + +## output tag=sensor.** to Wendelin + + @type wendelin + streamtool_uri http://{{local_ip}}:4443 + buffer_type memory + buffer_chunk_limit 1MB + flush_interval 1s + num_threads 1 + use_keep_alive true + + + + @type wendelin + streamtool_uri http://{{local_ip}}:4443/different_uri/ + buffer_type memory + buffer_chunk_limit 1MB + flush_interval 1s + num_threads 1 + use_keep_alive true + + + + +# ---- monitoring & debugging ---- + + +# Listen HTTP for monitoring +# http://localhost:24220/api/plugins +# http://localhost:24220/api/plugins?type=TYPE +# http://localhost:24220/api/plugins?tag=MYTAG + + @type monitor_agent + @id monitor_agent_input + + port 5440 + + +# Listen DRb for debug + + @type debug_agent + @id debug_agent_input + + bind {{local_ip}} + port 5443 + + + + @type stdout + @id stdout_output + diff --git a/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in b/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in new file mode 100644 index 0000000000000000000000000000000000000000..6a68830d4fd942f74eb2d59716850fc79eac23a1 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in @@ -0,0 +1,5 @@ +#!${dash-output:dash} +# BEWARE: This file is operated by slapgrid +# BEWARE: It will be overwritten automatically + +{{ caddy_exec }} -conf {{ caddy_file }} -pidfile {{pidfile_dir}}/caddy_pidfile \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/tests/README.md b/software/erp5testnode/testsuite/fluentTest/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e3189e54b01396447cd191beadcea2b66bb36eb --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/tests/README.md @@ -0,0 +1 @@ +Test for fluentd wendelin plugin with keep-alive connection and caddy. \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/tests/setup.py b/software/erp5testnode/testsuite/fluentTest/tests/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..7dc5ddef0618f8b31cdb56c61ec2c5ee42596f04 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/tests/setup.py @@ -0,0 +1,51 @@ +############################################################################## +# +# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from setuptools import setup, find_packages +import glob +import os + +version = '0.0.1.dev0' +name = 'slapos.test.fluentTest' +long_description = open("README.md").read() + +setup(name=name, + version=version, + description="Test for fluentd wendelin plugin", + long_description=long_description, + long_description_content_type='text/markdown', + maintainer="Nexedi", + maintainer_email="info@nexedi.com", + url="https://lab.nexedi.com/nexedi/slapos", + packages=find_packages(), + install_requires=[ + 'slapos.core', + 'erp5.util', + 'requests', + ], + zip_safe=True, + test_suite='test', + ) \ No newline at end of file diff --git a/software/erp5testnode/testsuite/fluentTest/tests/test.py b/software/erp5testnode/testsuite/fluentTest/tests/test.py new file mode 100644 index 0000000000000000000000000000000000000000..10bf65151495fec3e995c77cc6e4b1d10cdc7a14 --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/tests/test.py @@ -0,0 +1,227 @@ +import unittest +import requests +from StringIO import StringIO + +import SimpleHTTPServer +import SocketServer +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import os + +import time +import utils +import threading +import subprocess +import psutil + +test_msg = "dummyInputSimpleIngest" +caddy_pidfile = os.environ.get('CADDY_DIR') +with open(caddy_pidfile) as f: + caddy_pid = f.readline() + +if os.environ.get('DEBUG'): + import logging + logging.basicConfig(level=logging.DEBUG) + import unittest + unittest.installHandler() + +class FluentdPluginTestCase(utils.SlapOSInstanceTestCase): + @classmethod + def getSoftwareURLList(cls): + return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), ) + +class TestServerHandler(BaseHTTPRequestHandler): + + posted_data = None + all_data = [] + request_tag = "" + + def _set_headers(self): + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + + def do_GET(self): + self._set_headers() + self.wfile.write("

hi!

") + + def do_HEAD(self): + self._set_headers() + + def do_POST(self): + content_length = int(self.headers['Content-Length']) # <--- Gets the size of data + post_data = self.rfile.read(content_length) # <--- Gets the data itself + + self._set_headers() + self.wfile.write(post_data) + + TestServerHandler.posted_data = post_data.split(" ")[1] + + TestServerHandler.all_data.append(post_data.split(" ")[1]) + + TestServerHandler.request_tag = find_tag(self.requestline,"=", " ") + +class TestIngestion(FluentdPluginTestCase): + + @classmethod + def startServer(cls): + port=9443 + server_address = (os.environ['LOCAL_IPV4'], port) + cls.server = HTTPServer(server_address, TestServerHandler) + cls.thread = threading.Thread(target=cls.server.serve_forever) + cls.thread.start() + print("server start") + + @classmethod + def stopServer(cls): + cls.server.shutdown() + cls.server.server_close() + print("serever shutdown") + + def setUp(self): + self.caddy_process = psutil.Process(int(caddy_pid)) + self.startServer() + + def tearDown(self): + self.stopServer() + + def test_get_request_responds_with_200(self): + ''' + simple get request to be sure that server is up + ''' + print("############## TEST 1 ##############") + print("conn_before = %s" %str(self.caddy_process.connections())) + url = self.computer_partition.getConnectionParameterDict()['url'] + resp = requests.get(url) + self.assertEqual(resp.status_code, 200) + print (resp.status_code) + + def test_simple_ingest(self): + + print("############## TEST 2 ##############") + start_fluentd_cat(self, test_msg, "tag_test_2") + time.sleep(10) + self.assertEqual(test_msg, TestServerHandler.posted_data) + + def test_keepAlive_on(self): + print("############## TEST 3 ##############") + + start_fluentd_cat(self, "dummyInputDelayCheckKeepAlive", "tag_test_3") + time.sleep(20) + + self.assertEqual( + ['ESTABLISHED'], + [conn.status for conn in self.caddy_process.connections('inet') + if len(conn.raddr) > 1 and conn.laddr.port == 4443]) + print("conn_after = %s" %str(self.caddy_process.connections())) + + + def test_ingest_with_15mins_delay(self): + ''' + sleep 15mins to test that connections doesn't break after long delay + and data is ingested correctly after the delay. + ''' + print("############## TEST 4 ##############") + + time.sleep(900) + start_fluentd_cat(self, "dummyInputDelay", "tag_test_4") + time.sleep(15) + self.assertEqual("dummyInputDelay", TestServerHandler.posted_data) + + def test_ingest_while_server_breakage(self): + ''' + stop and then start caddy again to check that + fluentd plugin keeps message in a local buffer + and correctly sends them when caddy is back online + ''' + print("############## TEST 5 ##############") + + start_fluentd_cat(self, "dummyInputCaddyRestart1", "tag_test_5_1") + time.sleep(10) + + kill_caddy(caddy_pid) + time.sleep(10) + + start_fluentd_cat(self, "dummyInputCaddyRestart2 ", "tag_test_5_2") + time.sleep(10) + start_fluentd_cat(self, "dummyInputCaddyRestart3 ", "tag_test_5_3") + time.sleep(10) + start_fluentd_cat(self, "dummyInputCaddyRestart4 ", "tag_test_5_4") + time.sleep(130) + + start_caddy(caddy_pid) + time.sleep(15) + + self.assertTrue("dummyInputCaddyRestart1" in TestServerHandler.all_data) + self.assertTrue("dummyInputCaddyRestart2" in TestServerHandler.all_data) + self.assertTrue("dummyInputCaddyRestart3" in TestServerHandler.all_data) + self.assertTrue("dummyInputCaddyRestart4" in TestServerHandler.all_data) + + def test_ingest_with_diff_tags(self): + ''' + ingest data with different tags + ''' + print("############## TEST 6 ##############") + + start_fluentd_cat(self, "dummyInputTags_6_1", "tag_Test_6_1") + time.sleep(10) + self.assertEqual("tag_Test_6_1", TestServerHandler.request_tag) + + start_fluentd_cat(self, "dummyInputTags_6_2", "tag_Test_6_2") + time.sleep(10) + self.assertEqual("tag_Test_6_2", TestServerHandler.request_tag) + + start_fluentd_cat(self, "dummyInputTags_6_3", "tag_Test_6_3") + time.sleep(10) + self.assertEqual("tag_Test_6_3", TestServerHandler.request_tag) + + def test_simple_ingest_on_different_uri(self): + print("############## TEST 7 ##############") + kill_caddy(caddy_pid) + time.sleep(5) + start_caddy(caddy_pid) + + + start_fluentd_cat(self, "dummyInputOnURI1", "tag_uri_1") + start_fluentd_cat(self, "dummyInputOnURI2", "different_tag_uri_2") + time.sleep(5) + self.assertTrue("dummyInputOnURI1" in TestServerHandler.all_data) + self.assertTrue("dummyInputOnURI2" in TestServerHandler.all_data) + self.assertEqual( + ['ESTABLISHED', 'ESTABLISHED'], + [conn.status for conn in self.caddy_process.connections('inet') + if len(conn.raddr) > 1 and conn.laddr.port == 4443]) + print("conn_after = %s" %str(self.caddy_process.connections())) + + +def start_fluentd_cat(self, test_msg, tag): + """Feeds `test_msg` with `tag` to fluentd. + """ + fluent_service = os.environ.get('FLUENT_SERVICE') + proc = subprocess.Popen( + [fluent_service + '/bin/fluent-cat', + '--none', + tag, + '-p', + '5438', + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={"GEM_PATH": fluent_service + "/lib/ruby/gems/1.8/" }) + stdout, stderr = proc.communicate("+ " + test_msg) + self.assertEqual(0, proc.wait()) + self.assertEqual('', stdout) + self.assertEqual('', stderr) + +def kill_caddy(caddy_pid): + + os.system("kill -TSTP %s" % caddy_pid) + print("Caddy is stopped.") + +def start_caddy(caddy_pid): + + os.system("kill -CONT %s" % caddy_pid) + print("Caddy is restarted.") + +def find_tag(s, start, end): + return (s.split(start))[1].split(end)[0] diff --git a/software/erp5testnode/testsuite/fluentTest/tests/utils.py b/software/erp5testnode/testsuite/fluentTest/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f15f1df8b6044b95cdc7623776ca0eb391e38edf --- /dev/null +++ b/software/erp5testnode/testsuite/fluentTest/tests/utils.py @@ -0,0 +1,263 @@ +############################################################################## +# +# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import unittest +import os +import socket +from contextlib import closing +import logging +import StringIO +import xmlrpclib +import supervisor.xmlrpc +from erp5.util.testnode.SlapOSControler import SlapOSControler +from erp5.util.testnode.ProcessManager import ProcessManager +# Utility functions +def findFreeTCPPort(ip=''): + """Find a free TCP port to listen to. + """ + family = socket.AF_INET6 if ':' in ip else socket.AF_INET + with closing(socket.socket(family, socket.SOCK_STREAM)) as s: + s.bind((ip, 0)) + return s.getsockname()[1] +# TODO: +# - allow requesting multiple instances ? +class SlapOSInstanceTestCase(unittest.TestCase): + """Install one slapos instance. + This test case install software(s) and request one instance during `setUpClass` + and destroy the instance during `tearDownClass`. + Software Release URL, Instance Software Type and Instance Parameters can be defined + on the class. + All tests from the test class will run with the same instance. + The following class attributes are available: + * `computer_partition`: the computer partition instance, implementing + `slapos.slap.interface.slap.IComputerPartition`. + * `computer_partition_root_path`: the path of the instance root directory. + """ + # Methods to be defined by subclasses. + @classmethod + def getSoftwareURLList(cls): + """Return URL of software releases to install. + To be defined by subclasses. + """ + raise NotImplementedError() + @classmethod + def getInstanceParameterDict(cls): + """Return instance parameters + To be defined by subclasses if they need to request instance with specific + parameters. + """ + return {} + @classmethod + def getInstanceSoftwareType(cls): + """Return software type for instance, default "default" + To be defined by subclasses if they need to request instance with specific + software type. + """ + return "default" + # Utility methods. + def getSupervisorRPCServer(self): + """Returns a XML-RPC connection to the supervisor used by slapos node + Refer to http://supervisord.org/api.html for details of available methods. + """ + # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664 + return xmlrpclib.ServerProxy( + 'http://slapos-supervisor', + transport=supervisor.xmlrpc.SupervisorTransport( + None, + None, + # XXX hardcoded socket path + serverurl="unix://{working_directory}/inst/supervisord.socket".format( + **self.config))) + # Unittest methods + @classmethod + def setUpClass(cls): + """Setup the class, build software and request an instance. + If you have to override this method, do not forget to call this method on + parent class. + """ + try: + cls.setUpWorkingDirectory() + cls.setUpConfig() + cls.setUpSlapOSController() + cls.runSoftwareRelease() + # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage: + # cls.supplySoftwares() + # cls.installSoftwares() + cls.runComputerPartition() + # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage: + # cls.requestInstances() + # cls.createInstances() + # cls.requestInstances() + except Exception: + cls.stopSlapOSProcesses() + raise + @classmethod + def tearDownClass(cls): + """Tear down class, stop the processes and destroy instance. + """ + cls.stopSlapOSProcesses() + # Implementation + @classmethod + def stopSlapOSProcesses(cls): + if hasattr(cls, '_process_manager'): + cls._process_manager.killPreviousRun() + @classmethod + def setUpWorkingDirectory(cls): + """Initialise the directories""" + cls.working_directory = os.environ.get( + 'SLAPOS_TEST_WORKING_DIR', + os.path.join(os.path.dirname(__file__), '.slapos')) + # To prevent error: Cannot open an HTTP server: socket.error reported + # AF_UNIX path too long This `working_directory` should not be too deep. + # Socket path is 108 char max on linux + # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238 + # Supervisord socket name contains the pid number, which is why we add + # .xxxxxxx in this check. + if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108: + raise RuntimeError('working directory ( {} ) is too deep, try setting ' + 'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory)) + if not os.path.exists(cls.working_directory): + os.mkdir(cls.working_directory) + @classmethod + def setUpConfig(cls): + """Create slapos configuration""" + cls.config = { + "working_directory": cls.working_directory, + "slapos_directory": cls.working_directory, + "log_directory": cls.working_directory, + "computer_id": 'slapos.test', # XXX + 'proxy_database': os.path.join(cls.working_directory, 'proxy.db'), + 'partition_reference': cls.__name__, + # "proper" slapos command must be in $PATH + 'slapos_binary': 'slapos', + } + # Some tests are expecting that local IP is not set to 127.0.0.1 + ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1') + ipv6_address = os.environ['GLOBAL_IPV6'] + cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address + cls.config['ipv6_address'] = ipv6_address + cls.config['proxy_port'] = findFreeTCPPort(ipv4_address) + cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format( + **cls.config) + @classmethod + def setUpSlapOSController(cls): + """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`. + This is equivalent to: + slapos proxy start + for sr in getSoftwareURLList; do + slapos supply $SR $COMP + done + """ + cls._process_manager = ProcessManager() + # XXX this code is copied from testnode code + cls.slapos_controler = SlapOSControler( + cls.working_directory, + cls.config + ) + slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log') + logger = logging.getLogger(__name__) + logger.debug('Configured slapproxy log to %r', slapproxy_log) + cls.software_url_list = cls.getSoftwareURLList() + cls.slapos_controler.initializeSlapOSControler( + slapproxy_log=slapproxy_log, + process_manager=cls._process_manager, + reset_software=False, + software_path_list=cls.software_url_list) + # XXX we should check *earlier* if that pidfile exist and if supervisord + # process still running, because if developer started supervisord (or bugs?) + # then another supervisord will start and starting services a second time + # will fail. + cls._process_manager.supervisord_pid_file = os.path.join( + cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid') + @classmethod + def runSoftwareRelease(cls): + """Run all the software releases that were supplied before. + This is the equivalent of `slapos node software`. + The tests will be marked file if software building fail. + """ + logger = logging.getLogger() + logger.level = logging.DEBUG + stream = StringIO.StringIO() + stream_handler = logging.StreamHandler(stream) + logger.addHandler(stream_handler) + try: + cls.software_status_dict = cls.slapos_controler.runSoftwareRelease( + cls.config, environment=os.environ) + stream.seek(0) + stream.flush() + message = ''.join(stream.readlines()[-100:]) + assert cls.software_status_dict['status_code'] == 0, message + finally: + logger.removeHandler(stream_handler) + del stream + @classmethod + def runComputerPartition(cls): + """Instanciate the software. + This is the equivalent of doing: + slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict + slapos node instance + and return the slapos request instance parameters. + This can be called by tests to simulate re-request with different parameters. + """ + logger = logging.getLogger() + logger.level = logging.DEBUG + stream = StringIO.StringIO() + stream_handler = logging.StreamHandler(stream) + logger.addHandler(stream_handler) + if cls.getInstanceSoftwareType() != 'default': + raise NotImplementedError + instance_parameter_dict = cls.getInstanceParameterDict() + try: + cls.instance_status_dict = cls.slapos_controler.runComputerPartition( + cls.config, + cluster_configuration=instance_parameter_dict, + environment=os.environ) + stream.seek(0) + stream.flush() + message = ''.join(stream.readlines()[-100:]) + assert cls.instance_status_dict['status_code'] == 0, message + finally: + logger.removeHandler(stream_handler) + del stream + # FIXME: similar to test node, only one (root) partition is really + # supported for now. + computer_partition_list = [] + for i in range(len(cls.software_url_list)): + computer_partition_list.append( + cls.slapos_controler.slap.registerOpenOrder().request( + cls.software_url_list[i], + # This is how testnode's SlapOSControler name created partitions + partition_reference='testing partition {i}'.format( + i=i, **cls.config), + partition_parameter_kw=instance_parameter_dict)) + # expose some class attributes so that tests can use them: + # the ComputerPartition instances, to getInstanceParameterDict + cls.computer_partition = computer_partition_list[0] + # the path of the instance on the filesystem, for low level inspection + cls.computer_partition_root_path = os.path.join( + cls.config['working_directory'], + 'inst', + cls.computer_partition.getId())