diff --git a/component/bzip2/buildout.cfg b/component/bzip2/buildout.cfg
index 7633a9efa58213f05c844c31e983bf415506731c..a27ecb5a258899ae1db8980aed4e3916c940d7d4 100644
--- a/component/bzip2/buildout.cfg
+++ b/component/bzip2/buildout.cfg
@@ -4,7 +4,7 @@ parts =
[bzip2]
recipe = slapos.recipe.cmmi
-url = http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
+url = https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz
md5sum = 00b516f4704d4a7cb50a1d97e6e8e15b
configure-command = true
make-options =
diff --git a/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg b/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..bbad91df54ae8bb077bc3378fc605c7ce97b387c
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg
@@ -0,0 +1,35 @@
+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+# But avoid directories, they are not portable.
+# Copied verbatim.
+# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+# by the re-generation script.
+# Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-profile]
+filename = instance.cfg.in
+md5sum = 45c4adaae557bed5c25c24f4c4506b3d
+
+[template-fluentd-config]
+filename = templates/fluentd-conf.cnf.in
+md5sum = 46eb9943800fe5909953042cef596b6c
+
+[template-caddy-service]
+filename = templates/template-caddy-service.sh.in
+md5sum = 3e09969c479931ca4df6546abbd1a117
+
+[template-caddyfile]
+filename = templates/Caddyfile.in
+md5sum = 8644e98038ab76a8e5a499e22f92660a
+
+[template-caddy-fluentd]
+filename = instance-caddy-fluentd.cfg.in
+md5sum = a022edfb2b08e8376b7f4140f8d87917
diff --git a/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in b/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..77342aac9e4ab63ac987baf38f51430d45f627b5
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in
@@ -0,0 +1,109 @@
+[buildout]
+parts =
+ slapos-test-runner
+ directory
+ fluentd-config
+ fluentd-service
+ caddy-service
+ publish-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+run = $${:var}/run
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promise = $${:etc}/promise/
+www = $${:srv}/www
+home = $${:etc}/home
+ssl = $${:etc}/ssl
+working-dir = $${buildout:directory}/tmp/
+
+#################################
+# fluentd service
+#################################
+
+[fluentd-config]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fluentd-config:output}
+rendered = $${directory:etc}/fluentd-conf.cnf
+mode = 0600
+context =
+ key local_ip caddy-configuration:local_ip
+
+[fluentd-service]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/fluentd-service
+path = ${fluentd:location}
+command-line = ${fluentd:location}/bin/fluentd -vv
+ -c $${directory:etc}/fluentd-conf.cnf -p ${fluentd-plugin-dev-repository:location}/lib/fluent/plugin/ -o $${directory:etc}/fluent.log
+environment =
+ GEM_PATH= ${fluentd:location}/lib/ruby/gems/1.8/
+output = $${:wrapper-path}
+
+#################################
+# caddy service
+#################################
+
+[caddy-service]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-service:output}
+rendered = $${directory:services}/caddy
+mode = 0700
+context =
+ key caddy_exec caddy-exec-dict:caddy-exec-file
+ key caddy_file caddy-configuration:rendered
+ key pidfile_dir directory:etc
+
+[caddy-exec-dict]
+caddy-exec-file = ${gowork:bin}/caddy
+
+[caddy-configuration]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddyfile:output}
+rendered = $${directory:etc}/Caddyfile
+mode = 0600
+access_log = $${directory:log}/caddy-access.log
+error_log = $${directory:log}/caddy-error.log
+local_ip = $${slap-network-information:local-ipv4}
+context =
+ key local_ip caddy-configuration:local_ip
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+url = http://$${caddy-configuration:local_ip}:4443
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/runTestSuite
+command-line =
+ ${buildout:bin-directory}/runTestSuite
+ --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+ --source_code_path_list=$${slapos:location}/software/erp5testnode/testsuite/fluentTest/tests/
+environment =
+ PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+ LOCAL_IPV4=$${slap-network-information:local-ipv4}
+ GLOBAL_IPV6=$${slap-network-information:global-ipv6}
+ FLUENT_SERVICE = $${fluentd-service:path}
+ SLAPOS_TEST_WORKING_DIR=$${directory:working-dir}
+ CADDY_DIR = $${directory:etc}/caddy_pidfile
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/instance.cfg.in b/software/erp5testnode/testsuite/fluentTest/instance.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..b31974dfeb6a135af2a1f80a060f01a91cfb16a7
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/instance.cfg.in
@@ -0,0 +1,51 @@
+#############################
+#
+# Deploy caddy instance
+#
+#############################
+[buildout]
+parts =
+ switch-softwaretype
+
+# publish-connection-parameter
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:caddy}
+caddy = $${dynamic-template-caddy-fluentd:rendered}
+
+[dynamic-template-caddy-fluentd]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-fluentd:output}
+rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+filename = instance-caddy-fluentd.cfg
+
+[slap-connection]
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+[instance-parameter]
+# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
+# We use the slapconfiguration recipe with a few parameters (partition id,
+# computer id, certificate, etc).
+# It will then authenticate to SlapOS Master and fetch the instance parameters.
+# The parameters are accessible from {instance-parameter:configuration.name-of-parameter}
+# Always the same. Just copy/paste.
+# See docstring of slapos.cookbook:slapconfiguration for more information.
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/software.cfg b/software/erp5testnode/testsuite/fluentTest/software.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..a66a12d3b0980f2562385d6c0735778a01e13dd1
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/software.cfg
@@ -0,0 +1,125 @@
+[buildout]
+
+extends =
+ ../../../../stack/slapos.cfg
+ ../../../../software/caddy/software.cfg
+ ../../../../software/fluentd/software.cfg
+ buildout.hash.cfg
+
+index = https://pypi.python.org/simple/
+
+parts =
+ slapos-cookbook
+ eggs
+ caddy
+ fluentd
+ fluentd-plugin-dev-repository
+ instance-profile
+ template-fluentd-config
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.fluentTest-setup]
+<= setup-develop-egg
+egg = slapos.test.fluentTest
+setup = ${slapos-repository:location}/software/erp5testnode/testsuite/fluentTest/tests/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+ ${slapos.test.fluentTest-setup:egg}
+ ${erp5.util-setup:egg}
+ slapos.core
+entry-points =
+ runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+ runTestSuite
+ slapos
+interpreter=
+ python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+#repository = https://lab.nexedi.com/nexedi/slapos.git
+repository = https://lab.nexedi.com/Sokhoyan/slapos.git
+branch = FluentTest_withFixUtil
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 3b9d23bff23cfba208131b004ae4a0e7484e9aff
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.fluentTest =
+erp5.util =
+#erp5.util = 0.4.51
+
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+mode = 0644
+
+[template-fluentd-config]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/fluentd-conf.cnf.in
+mode = 0644
+
+[fluentd-plugin-dev-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/fluent-plugin-wendelin.git
+# dir is pretty name as top-level -dev recipe
+git-executable = ${git:location}/bin/git
+
+[fluentd]
+gems =
+ fluentd==0.14.14
+ fluent-plugin-td
+ bundler
+ fluent-plugin-bin
+
+[template-caddy-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template-caddy-service.sh.in
+mode = 0644
+
+[template-caddyfile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/Caddyfile.in
+mode = 0644
+
+[template-caddy-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
+mode = 0644
+
+[caddy]
+recipe = slapos.recipe.cmmi
+path = ${go_github.com_mholt_caddy:location}
+go = ${gowork:golang}/bin/go
+configure-command = :
+make-targets =
+make-binary = cd ${:path}/caddy && ${:go} install -v
+environment =
+ PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
+ GOPATH=${gowork:directory}
+output = ${gowork:bin}/caddy
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in b/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in
new file mode 100644
index 0000000000000000000000000000000000000000..a885d76120719a23e33ee5d696881962979d66de
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/templates/Caddyfile.in
@@ -0,0 +1,4 @@
+{{local_ip}}:4443
+
+ proxy / {{local_ip}}:9443
+
diff --git a/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in b/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in
new file mode 100644
index 0000000000000000000000000000000000000000..bec70555184466e76099c3904978fc063fde3f33
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in
@@ -0,0 +1,55 @@
+
+
+## output tag=sensor.** to Wendelin
+
+ @type wendelin
+ streamtool_uri http://{{local_ip}}:4443
+ buffer_type memory
+ buffer_chunk_limit 1MB
+ flush_interval 1s
+ num_threads 1
+ use_keep_alive true
+
+
+
+ @type wendelin
+ streamtool_uri http://{{local_ip}}:4443/different_uri/
+ buffer_type memory
+ buffer_chunk_limit 1MB
+ flush_interval 1s
+ num_threads 1
+ use_keep_alive true
+
+
+
+
+# ---- monitoring & debugging ----
+
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+
+
+# Listen DRb for debug
+
+
+
+ @type stdout
+ @id stdout_output
+
diff --git a/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in b/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in
new file mode 100644
index 0000000000000000000000000000000000000000..6a68830d4fd942f74eb2d59716850fc79eac23a1
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/templates/template-caddy-service.sh.in
@@ -0,0 +1,5 @@
+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+{{ caddy_exec }} -conf {{ caddy_file }} -pidfile {{pidfile_dir}}/caddy_pidfile
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/tests/README.md b/software/erp5testnode/testsuite/fluentTest/tests/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..9e3189e54b01396447cd191beadcea2b66bb36eb
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/tests/README.md
@@ -0,0 +1 @@
+Test for fluentd wendelin plugin with keep-alive connection and caddy.
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/tests/setup.py b/software/erp5testnode/testsuite/fluentTest/tests/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..7dc5ddef0618f8b31cdb56c61ec2c5ee42596f04
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/tests/setup.py
@@ -0,0 +1,51 @@
+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.fluentTest'
+long_description = open("README.md").read()
+
+setup(name=name,
+ version=version,
+ description="Test for fluentd wendelin plugin",
+ long_description=long_description,
+ long_description_content_type='text/markdown',
+ maintainer="Nexedi",
+ maintainer_email="info@nexedi.com",
+ url="https://lab.nexedi.com/nexedi/slapos",
+ packages=find_packages(),
+ install_requires=[
+ 'slapos.core',
+ 'erp5.util',
+ 'requests',
+ ],
+ zip_safe=True,
+ test_suite='test',
+ )
\ No newline at end of file
diff --git a/software/erp5testnode/testsuite/fluentTest/tests/test.py b/software/erp5testnode/testsuite/fluentTest/tests/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..10bf65151495fec3e995c77cc6e4b1d10cdc7a14
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/tests/test.py
@@ -0,0 +1,227 @@
+import unittest
+import requests
+from StringIO import StringIO
+
+import SimpleHTTPServer
+import SocketServer
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import os
+
+import time
+import utils
+import threading
+import subprocess
+import psutil
+
+test_msg = "dummyInputSimpleIngest"
+caddy_pidfile = os.environ.get('CADDY_DIR')
+with open(caddy_pidfile) as f:
+ caddy_pid = f.readline()
+
+if os.environ.get('DEBUG'):
+ import logging
+ logging.basicConfig(level=logging.DEBUG)
+ import unittest
+ unittest.installHandler()
+
+class FluentdPluginTestCase(utils.SlapOSInstanceTestCase):
+ @classmethod
+ def getSoftwareURLList(cls):
+ return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+class TestServerHandler(BaseHTTPRequestHandler):
+
+ posted_data = None
+ all_data = []
+ request_tag = ""
+
+ def _set_headers(self):
+ self.send_response(200)
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+
+ def do_GET(self):
+ self._set_headers()
+ self.wfile.write("
hi!
")
+
+ def do_HEAD(self):
+ self._set_headers()
+
+ def do_POST(self):
+ content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+ post_data = self.rfile.read(content_length) # <--- Gets the data itself
+
+ self._set_headers()
+ self.wfile.write(post_data)
+
+ TestServerHandler.posted_data = post_data.split(" ")[1]
+
+ TestServerHandler.all_data.append(post_data.split(" ")[1])
+
+ TestServerHandler.request_tag = find_tag(self.requestline,"=", " ")
+
+class TestIngestion(FluentdPluginTestCase):
+
+ @classmethod
+ def startServer(cls):
+ port=9443
+ server_address = (os.environ['LOCAL_IPV4'], port)
+ cls.server = HTTPServer(server_address, TestServerHandler)
+ cls.thread = threading.Thread(target=cls.server.serve_forever)
+ cls.thread.start()
+ print("server start")
+
+ @classmethod
+ def stopServer(cls):
+ cls.server.shutdown()
+ cls.server.server_close()
+ print("serever shutdown")
+
+ def setUp(self):
+ self.caddy_process = psutil.Process(int(caddy_pid))
+ self.startServer()
+
+ def tearDown(self):
+ self.stopServer()
+
+ def test_get_request_responds_with_200(self):
+ '''
+ simple get request to be sure that server is up
+ '''
+ print("############## TEST 1 ##############")
+ print("conn_before = %s" %str(self.caddy_process.connections()))
+ url = self.computer_partition.getConnectionParameterDict()['url']
+ resp = requests.get(url)
+ self.assertEqual(resp.status_code, 200)
+ print (resp.status_code)
+
+ def test_simple_ingest(self):
+
+ print("############## TEST 2 ##############")
+ start_fluentd_cat(self, test_msg, "tag_test_2")
+ time.sleep(10)
+ self.assertEqual(test_msg, TestServerHandler.posted_data)
+
+ def test_keepAlive_on(self):
+ print("############## TEST 3 ##############")
+
+ start_fluentd_cat(self, "dummyInputDelayCheckKeepAlive", "tag_test_3")
+ time.sleep(20)
+
+ self.assertEqual(
+ ['ESTABLISHED'],
+ [conn.status for conn in self.caddy_process.connections('inet')
+ if len(conn.raddr) > 1 and conn.laddr.port == 4443])
+ print("conn_after = %s" %str(self.caddy_process.connections()))
+
+
+ def test_ingest_with_15mins_delay(self):
+ '''
+ sleep 15mins to test that connections doesn't break after long delay
+ and data is ingested correctly after the delay.
+ '''
+ print("############## TEST 4 ##############")
+
+ time.sleep(900)
+ start_fluentd_cat(self, "dummyInputDelay", "tag_test_4")
+ time.sleep(15)
+ self.assertEqual("dummyInputDelay", TestServerHandler.posted_data)
+
+ def test_ingest_while_server_breakage(self):
+ '''
+ stop and then start caddy again to check that
+ fluentd plugin keeps message in a local buffer
+ and correctly sends them when caddy is back online
+ '''
+ print("############## TEST 5 ##############")
+
+ start_fluentd_cat(self, "dummyInputCaddyRestart1", "tag_test_5_1")
+ time.sleep(10)
+
+ kill_caddy(caddy_pid)
+ time.sleep(10)
+
+ start_fluentd_cat(self, "dummyInputCaddyRestart2 ", "tag_test_5_2")
+ time.sleep(10)
+ start_fluentd_cat(self, "dummyInputCaddyRestart3 ", "tag_test_5_3")
+ time.sleep(10)
+ start_fluentd_cat(self, "dummyInputCaddyRestart4 ", "tag_test_5_4")
+ time.sleep(130)
+
+ start_caddy(caddy_pid)
+ time.sleep(15)
+
+ self.assertTrue("dummyInputCaddyRestart1" in TestServerHandler.all_data)
+ self.assertTrue("dummyInputCaddyRestart2" in TestServerHandler.all_data)
+ self.assertTrue("dummyInputCaddyRestart3" in TestServerHandler.all_data)
+ self.assertTrue("dummyInputCaddyRestart4" in TestServerHandler.all_data)
+
+ def test_ingest_with_diff_tags(self):
+ '''
+ ingest data with different tags
+ '''
+ print("############## TEST 6 ##############")
+
+ start_fluentd_cat(self, "dummyInputTags_6_1", "tag_Test_6_1")
+ time.sleep(10)
+ self.assertEqual("tag_Test_6_1", TestServerHandler.request_tag)
+
+ start_fluentd_cat(self, "dummyInputTags_6_2", "tag_Test_6_2")
+ time.sleep(10)
+ self.assertEqual("tag_Test_6_2", TestServerHandler.request_tag)
+
+ start_fluentd_cat(self, "dummyInputTags_6_3", "tag_Test_6_3")
+ time.sleep(10)
+ self.assertEqual("tag_Test_6_3", TestServerHandler.request_tag)
+
+ def test_simple_ingest_on_different_uri(self):
+ print("############## TEST 7 ##############")
+ kill_caddy(caddy_pid)
+ time.sleep(5)
+ start_caddy(caddy_pid)
+
+
+ start_fluentd_cat(self, "dummyInputOnURI1", "tag_uri_1")
+ start_fluentd_cat(self, "dummyInputOnURI2", "different_tag_uri_2")
+ time.sleep(5)
+ self.assertTrue("dummyInputOnURI1" in TestServerHandler.all_data)
+ self.assertTrue("dummyInputOnURI2" in TestServerHandler.all_data)
+ self.assertEqual(
+ ['ESTABLISHED', 'ESTABLISHED'],
+ [conn.status for conn in self.caddy_process.connections('inet')
+ if len(conn.raddr) > 1 and conn.laddr.port == 4443])
+ print("conn_after = %s" %str(self.caddy_process.connections()))
+
+
+def start_fluentd_cat(self, test_msg, tag):
+ """Feeds `test_msg` with `tag` to fluentd.
+ """
+ fluent_service = os.environ.get('FLUENT_SERVICE')
+ proc = subprocess.Popen(
+ [fluent_service + '/bin/fluent-cat',
+ '--none',
+ tag,
+ '-p',
+ '5438',
+ ],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env={"GEM_PATH": fluent_service + "/lib/ruby/gems/1.8/" })
+ stdout, stderr = proc.communicate("+ " + test_msg)
+ self.assertEqual(0, proc.wait())
+ self.assertEqual('', stdout)
+ self.assertEqual('', stderr)
+
+def kill_caddy(caddy_pid):
+
+ os.system("kill -TSTP %s" % caddy_pid)
+ print("Caddy is stopped.")
+
+def start_caddy(caddy_pid):
+
+ os.system("kill -CONT %s" % caddy_pid)
+ print("Caddy is restarted.")
+
+def find_tag(s, start, end):
+ return (s.split(start))[1].split(end)[0]
diff --git a/software/erp5testnode/testsuite/fluentTest/tests/utils.py b/software/erp5testnode/testsuite/fluentTest/tests/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f15f1df8b6044b95cdc7623776ca0eb391e38edf
--- /dev/null
+++ b/software/erp5testnode/testsuite/fluentTest/tests/utils.py
@@ -0,0 +1,263 @@
+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+# Utility functions
+def findFreeTCPPort(ip=''):
+ """Find a free TCP port to listen to.
+ """
+ family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+ with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+ s.bind((ip, 0))
+ return s.getsockname()[1]
+# TODO:
+# - allow requesting multiple instances ?
+class SlapOSInstanceTestCase(unittest.TestCase):
+ """Install one slapos instance.
+ This test case install software(s) and request one instance during `setUpClass`
+ and destroy the instance during `tearDownClass`.
+ Software Release URL, Instance Software Type and Instance Parameters can be defined
+ on the class.
+ All tests from the test class will run with the same instance.
+ The following class attributes are available:
+ * `computer_partition`: the computer partition instance, implementing
+ `slapos.slap.interface.slap.IComputerPartition`.
+ * `computer_partition_root_path`: the path of the instance root directory.
+ """
+ # Methods to be defined by subclasses.
+ @classmethod
+ def getSoftwareURLList(cls):
+ """Return URL of software releases to install.
+ To be defined by subclasses.
+ """
+ raise NotImplementedError()
+ @classmethod
+ def getInstanceParameterDict(cls):
+ """Return instance parameters
+ To be defined by subclasses if they need to request instance with specific
+ parameters.
+ """
+ return {}
+ @classmethod
+ def getInstanceSoftwareType(cls):
+ """Return software type for instance, default "default"
+ To be defined by subclasses if they need to request instance with specific
+ software type.
+ """
+ return "default"
+ # Utility methods.
+ def getSupervisorRPCServer(self):
+ """Returns a XML-RPC connection to the supervisor used by slapos node
+ Refer to http://supervisord.org/api.html for details of available methods.
+ """
+ # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+ return xmlrpclib.ServerProxy(
+ 'http://slapos-supervisor',
+ transport=supervisor.xmlrpc.SupervisorTransport(
+ None,
+ None,
+ # XXX hardcoded socket path
+ serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+ **self.config)))
+ # Unittest methods
+ @classmethod
+ def setUpClass(cls):
+ """Setup the class, build software and request an instance.
+ If you have to override this method, do not forget to call this method on
+ parent class.
+ """
+ try:
+ cls.setUpWorkingDirectory()
+ cls.setUpConfig()
+ cls.setUpSlapOSController()
+ cls.runSoftwareRelease()
+ # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+ # cls.supplySoftwares()
+ # cls.installSoftwares()
+ cls.runComputerPartition()
+ # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+ # cls.requestInstances()
+ # cls.createInstances()
+ # cls.requestInstances()
+ except Exception:
+ cls.stopSlapOSProcesses()
+ raise
+ @classmethod
+ def tearDownClass(cls):
+ """Tear down class, stop the processes and destroy instance.
+ """
+ cls.stopSlapOSProcesses()
+ # Implementation
+ @classmethod
+ def stopSlapOSProcesses(cls):
+ if hasattr(cls, '_process_manager'):
+ cls._process_manager.killPreviousRun()
+ @classmethod
+ def setUpWorkingDirectory(cls):
+ """Initialise the directories"""
+ cls.working_directory = os.environ.get(
+ 'SLAPOS_TEST_WORKING_DIR',
+ os.path.join(os.path.dirname(__file__), '.slapos'))
+ # To prevent error: Cannot open an HTTP server: socket.error reported
+ # AF_UNIX path too long This `working_directory` should not be too deep.
+ # Socket path is 108 char max on linux
+ # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+ # Supervisord socket name contains the pid number, which is why we add
+ # .xxxxxxx in this check.
+ if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+ raise RuntimeError('working directory ( {} ) is too deep, try setting '
+ 'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+ if not os.path.exists(cls.working_directory):
+ os.mkdir(cls.working_directory)
+ @classmethod
+ def setUpConfig(cls):
+ """Create slapos configuration"""
+ cls.config = {
+ "working_directory": cls.working_directory,
+ "slapos_directory": cls.working_directory,
+ "log_directory": cls.working_directory,
+ "computer_id": 'slapos.test', # XXX
+ 'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+ 'partition_reference': cls.__name__,
+ # "proper" slapos command must be in $PATH
+ 'slapos_binary': 'slapos',
+ }
+ # Some tests are expecting that local IP is not set to 127.0.0.1
+ ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+ ipv6_address = os.environ['GLOBAL_IPV6']
+ cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+ cls.config['ipv6_address'] = ipv6_address
+ cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+ cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+ **cls.config)
+ @classmethod
+ def setUpSlapOSController(cls):
+ """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+ This is equivalent to:
+ slapos proxy start
+ for sr in getSoftwareURLList; do
+ slapos supply $SR $COMP
+ done
+ """
+ cls._process_manager = ProcessManager()
+ # XXX this code is copied from testnode code
+ cls.slapos_controler = SlapOSControler(
+ cls.working_directory,
+ cls.config
+ )
+ slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+ logger = logging.getLogger(__name__)
+ logger.debug('Configured slapproxy log to %r', slapproxy_log)
+ cls.software_url_list = cls.getSoftwareURLList()
+ cls.slapos_controler.initializeSlapOSControler(
+ slapproxy_log=slapproxy_log,
+ process_manager=cls._process_manager,
+ reset_software=False,
+ software_path_list=cls.software_url_list)
+ # XXX we should check *earlier* if that pidfile exist and if supervisord
+ # process still running, because if developer started supervisord (or bugs?)
+ # then another supervisord will start and starting services a second time
+ # will fail.
+ cls._process_manager.supervisord_pid_file = os.path.join(
+ cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+ @classmethod
+ def runSoftwareRelease(cls):
+ """Run all the software releases that were supplied before.
+ This is the equivalent of `slapos node software`.
+ The tests will be marked file if software building fail.
+ """
+ logger = logging.getLogger()
+ logger.level = logging.DEBUG
+ stream = StringIO.StringIO()
+ stream_handler = logging.StreamHandler(stream)
+ logger.addHandler(stream_handler)
+ try:
+ cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+ cls.config, environment=os.environ)
+ stream.seek(0)
+ stream.flush()
+ message = ''.join(stream.readlines()[-100:])
+ assert cls.software_status_dict['status_code'] == 0, message
+ finally:
+ logger.removeHandler(stream_handler)
+ del stream
+ @classmethod
+ def runComputerPartition(cls):
+ """Instanciate the software.
+ This is the equivalent of doing:
+ slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+ slapos node instance
+ and return the slapos request instance parameters.
+ This can be called by tests to simulate re-request with different parameters.
+ """
+ logger = logging.getLogger()
+ logger.level = logging.DEBUG
+ stream = StringIO.StringIO()
+ stream_handler = logging.StreamHandler(stream)
+ logger.addHandler(stream_handler)
+ if cls.getInstanceSoftwareType() != 'default':
+ raise NotImplementedError
+ instance_parameter_dict = cls.getInstanceParameterDict()
+ try:
+ cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+ cls.config,
+ cluster_configuration=instance_parameter_dict,
+ environment=os.environ)
+ stream.seek(0)
+ stream.flush()
+ message = ''.join(stream.readlines()[-100:])
+ assert cls.instance_status_dict['status_code'] == 0, message
+ finally:
+ logger.removeHandler(stream_handler)
+ del stream
+ # FIXME: similar to test node, only one (root) partition is really
+ # supported for now.
+ computer_partition_list = []
+ for i in range(len(cls.software_url_list)):
+ computer_partition_list.append(
+ cls.slapos_controler.slap.registerOpenOrder().request(
+ cls.software_url_list[i],
+ # This is how testnode's SlapOSControler name created partitions
+ partition_reference='testing partition {i}'.format(
+ i=i, **cls.config),
+ partition_parameter_kw=instance_parameter_dict))
+ # expose some class attributes so that tests can use them:
+ # the ComputerPartition instances, to getInstanceParameterDict
+ cls.computer_partition = computer_partition_list[0]
+ # the path of the instance on the filesystem, for low level inspection
+ cls.computer_partition_root_path = os.path.join(
+ cls.config['working_directory'],
+ 'inst',
+ cls.computer_partition.getId())