prepare to run with setup.py

software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in

 [buildout]
 parts =
+  slapos-test-runner
   directory
-  runTestSuite-instance
   fluentdConfig
   fluentd-service
   caddy-service
@@ -27,17 +27,11 @@ www = $${:srv}/www
 home = $${:etc}/home
 ssl = $${:etc}/ssl
 
-[runTestSuite-instance]
-recipe = slapos.recipe.template
-url = ${template-runTestSuite:output}
-output = $${directory:bin}/runTestSuite
-buildout-directory = $${buildout:directory}
-mode = 0700
 
 [runTest-instance]
 recipe = slapos.recipe.template
 url = ${template-test:output}
-output = $${directory:bin}/testIngestion.py
+output = $${directory:bin}/test.py
 buildout-directory = $${buildout:directory}
 mode = 0700
 
@@ -93,4 +87,26 @@ context =
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-secure_access = http://$${caddy-configuration:local_ip}:4443}
\ No newline at end of file
+secure_access = http://$${caddy-configuration:local_ip}:4443}
+
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/erp5testnode/testsuite/fluentTest/tests/
+####################################################
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/software.cfg

@@ -14,9 +14,72 @@ parts =
   fluentd
   fluentd-plugin-dev-repository
   instance-profile
-  template-runTestSuite
   template-fluentdConfig
-  
+
+
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.fluentTest-setup]
+<= setup-develop-egg
+egg = slapos.test.fluentTest
+setup = ${slapos-repository:location}/software/erp5testnode/testsuite/fluentTest/tests/
+#setup = ${:_profile_base_location_}/tests/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.fluentTest-setup:egg}
+  ${erp5.util-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = testForFleuntdClean
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/Sokhoyan/slapos.git
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite 
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.fluentTest =
+erp5.util =
+#erp5.util = 0.4.51
+
+
+
+
+#[eggs]
+#recipe = zc.recipe.egg
+#eggs =
+#  erp5.util
+#  ${lxml-python:egg}
+#  requests
+#interpreter = pythonwitheggs
 
 [instance-profile]
 recipe = slapos.recipe.template
@@ -25,32 +88,17 @@ url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
 mode = 0644
 
-[template-runTestSuite]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/runTestSuite.in
-#md5sum = ff66d13f73982e8257eb5535cdb541c7
-output = ${buildout:directory}/runTestSuite.in
-mode = 0644
-
 [template-test]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/testIngestion.py
+url = ${:_profile_base_location_}/tests/test.py
 #md5sum = ff66d13f73982e8257eb5535cdb541c7
-output = ${buildout:directory}/testIngestion.py
+output = ${buildout:directory}/test.py
 mode = 0644
 
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  erp5.util
-  ${lxml-python:egg}
-  requests
-interpreter = pythonwitheggs
-
 [template-fluentdConfig]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/fluentd-conf.cnf.in
-md5sum = fd896793455e1d04082449212d285b55
+url = ${:_profile_base_location_}/templates/fluentd-conf.cnf.in
+md5sum = 63b8bea7fcc18af68e8bdf2afc33a92d
 filename = fluentd-conf.cnf.in
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
@@ -71,22 +119,22 @@ gems =
 
 [template-caddy-service]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/template-caddy-service.sh.in
+url = ${:_profile_base_location_}/templates/template-caddy-service.sh.in
 md5sum = 3e09969c479931ca4df6546abbd1a117
 output = ${buildout:directory}/template-caddy-service.sh.in
 mode = 0644
 
 [template-caddyfile]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/Caddyfile.in
-md5sum = 8644e98038ab76a8e5a499e22f92660a
+url = ${:_profile_base_location_}/templates/Caddyfile.in
+md5sum = 4cbb458590860cec705b48977718609f
 output = ${buildout:directory}/Caddyfile.in
 mode = 0644
 
 [template-caddy-fluentd]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-caddy-fluentd.cfg.in
-md5sum = 4e0c6e0464ad1b2f1adadda10771bb86
+md5sum = ac6f4d735716c1a4e5226f1dfad2966d
 output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
 mode = 0644
 
@@ -102,6 +150,6 @@ environment =
   GOPATH=${gowork:directory}
 output =  ${gowork:bin}/caddy
 
-[versions]
-slapos.recipe.template = 4.3
-erp5.util = 0.4.50
+#versions]
+#slapos.recipe.template = 4.3
+#erp5.util = 0.4.50

software/erp5testnode/testsuite/fluentTest/tests/README.md

+Test for fluentd wendelin plugin with keep-alive connection and caddy. 
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/tests/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.fluentTest'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for fluentd wendelin plugin",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'erp5.util',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/testIngestion.py → software/erp5testnode/testsuite/fluentTest/tests/test.py

@@ -13,6 +13,7 @@ import os
 
 import threading
 import time
+import utils
 
 
 test_msg = "dummyInputSimpleIngest"
@@ -24,6 +25,15 @@ posted_data = None
 all_data = []
 request_tag = ""
 
+###################################################
+
+class FluentdPluginTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+###################################################
+
 class TestServerHandler(BaseHTTPRequestHandler):
 
     def _set_headers(self):
@@ -56,7 +66,24 @@ class TestServerHandler(BaseHTTPRequestHandler):
         global request_tag
         request_tag = find_tag(self.requestline,"=", " ")
 
-class TestPost(unittest.TestCase):
+class TestIngestion(FluentdPluginTestCase):
+
+    def setUp(self):
+      
+      port=9443
+      server_address = ('$${slap-network-information:local-ipv4}', port)
+      httpd = HTTPServer(server_address, TestServerHandler)
+      thread = threading.Thread(target=httpd.serve_forever)
+      thread.start()
+      print 'Starting http...'
+      time.sleep(15)
+
+    def tearDown(self):
+      
+      httpd.shutdown()
+      print("all posted data : ")
+      print(all_data)
+    
 
     def test_1_get(self):
       print("############## TEST 1 ##############")
@@ -156,6 +183,8 @@ def start_caddy(caddy_pid):
 def find_tag(s, start, end):
   return (s.split(start))[1].split(end)[0]
 
+
+'''
 def main():
   
     port=9443
@@ -189,4 +218,5 @@ def main():
 
 if __name__ == "__main__":
   
-    main()
\ No newline at end of file
+    main()
+'''    
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/tests/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+import slapos
+
+
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+ # TODO: allow subclasses to request a specific software type ?
+
+
+  @classmethod
+  def setUpClass(cls):
+
+    working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    if len(working_directory + '/inst/supervisord.socket') > 108:
+      raise RuntimeError('working directory too deep, try setting SLAPOS_TEST_WORKING_DIR')
+
+    if not os.path.exists(working_directory):
+      os.mkdir(working_directory)
+
+    cls.config = config = {
+      "working_directory": working_directory,
+      "slapos_directory": working_directory,
+      "log_directory": working_directory,
+      "computer_id": 'slapos.test', # XXX
+      'proxy_database': os.path.join(working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    config['proxy_host'] = config['ipv4_address'] = ipv4_address
+    config['ipv6_address'] = ipv6_address
+    config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(**config)
+
+    cls._process_manager = process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    slapos_controler = SlapOSControler(
+        working_directory,
+        config
+    )
+
+    slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    software_url_list = cls.getSoftwareURLList()
+    slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=process_manager,
+        reset_software=False,
+        software_path_list=software_url_list)
+
+    process_manager.supervisord_pid_file = os.path.join(
+           slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+    software_status_dict = slapos_controler.runSoftwareRelease(config, environment=os.environ)
+    # TODO: log more details in this case
+    assert software_status_dict['status_code'] == 0
+
+    instance_parameter_dict = cls.getInstanceParmeterDict()
+    instance_status_dict = slapos_controler.runComputerPartition(
+        config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+    # TODO: log more details in this case
+    assert instance_status_dict['status_code'] == 0
+
+    # FIXME: similar to test node, only one (root) partition is really supported for now.
+    computer_partition_list = []
+    for i in range(len(software_url_list)):
+      computer_partition_list.append(
+          slapos_controler.slap.registerOpenOrder().request(
+          software_url_list[i],
+          # This is how testnode's SlapOSControler name created partitions
+          partition_reference='testing partition {i}'.format(i=i, **config),
+          partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParmeterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+  @classmethod
+  def tearDownClass(cls):
+    # FIXME: if setUpClass fail, this is not called and leaks zombie processes
+    cls._process_manager.killPreviousRun()