Commit 56d0240e authored by Eteri's avatar Eteri

squash!

remove unused parts, fix CameCase format

add buildout.hash.cfg for update-hash

remove unused my_server.py

remove global variables, change calling way of env varibale

use subprocess instead of os.system

rename tests, add comments

fix setUp and tearDown for tests

diable debug

get url from comp partition
parent ba2b7d3c
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = 45c4adaae557bed5c25c24f4c4506b3d
[template-fluentd-config]
filename = templates/fluentd-conf.cnf.in
md5sum = 88235b3b15cccc615acb6a9f9029684a
[template-caddy-service]
filename = templates/template-caddy-service.sh.in
md5sum = 3e09969c479931ca4df6546abbd1a117
[template-caddyfile]
filename = templates/Caddyfile.in
md5sum = 8644e98038ab76a8e5a499e22f92660a
[template-caddy-fluentd]
filename = instance-caddy-fluentd.cfg.in
md5sum = a022edfb2b08e8376b7f4140f8d87917
......@@ -2,11 +2,10 @@
parts =
slapos-test-runner
directory
fluentdConfig
fluentd-config
fluentd-service
caddy-service
publish-connection-information
runTest-instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -28,20 +27,13 @@ home = $${:etc}/home
ssl = $${:etc}/ssl
working-dir = $${buildout:directory}/tmp/
[runTest-instance]
recipe = slapos.recipe.template
url = ${template-test:output}
output = $${directory:bin}/test.py
buildout-directory = $${buildout:directory}
mode = 0700
#################################
# fluentd service
#################################
[fluentdConfig]
[fluentd-config]
recipe = slapos.recipe.template:jinja2
template = ${template-fluentdConfig:output}
template = ${template-fluentd-config:output}
rendered = $${directory:etc}/fluentd-conf.cnf
mode = 0600
context =
......@@ -87,7 +79,7 @@ context =
[publish-connection-information]
recipe = slapos.cookbook:publish
secure_access = http://$${caddy-configuration:local_ip}:4443}
url = http://$${caddy-configuration:local_ip}:4443
[download-source]
recipe = slapos.recipe.build:gitclone
......
......@@ -4,6 +4,7 @@ extends =
../../../../stack/slapos.cfg
../../../../software/caddy/software.cfg
../../../../software/fluentd/software.cfg
buildout.hash.cfg
index = https://pypi.python.org/simple/
......@@ -14,7 +15,7 @@ parts =
fluentd
fluentd-plugin-dev-repository
instance-profile
template-fluentdConfig
template-fluentd-config
[setup-develop-egg]
recipe = zc.recipe.egg:develop
......@@ -51,6 +52,7 @@ branch = master
[slapos-repository]
<= git-clone-repository
#repository = https://lab.nexedi.com/nexedi/slapos.git
repository = https://lab.nexedi.com/Sokhoyan/slapos.git
branch = FluentTest_withFixUtil
......@@ -69,22 +71,13 @@ erp5.util =
[instance-profile]
recipe = slapos.recipe.template
md5sum = 45c4adaae557bed5c25c24f4c4506b3d
url = ${:_profile_base_location_}/instance.cfg.in
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance.cfg
mode = 0644
[template-test]
[template-fluentd-config]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/tests/test.py
#md5sum = ff66d13f73982e8257eb5535cdb541c7
output = ${buildout:directory}/test.py
mode = 0644
[template-fluentdConfig]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/fluentd-conf.cnf.in
md5sum = f37d4d69895680abf40b847633876373
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/fluentd-conf.cnf.in
mode = 0644
......@@ -104,22 +97,19 @@ gems =
[template-caddy-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/template-caddy-service.sh.in
md5sum = 3e09969c479931ca4df6546abbd1a117
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-caddy-service.sh.in
mode = 0644
[template-caddyfile]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/Caddyfile.in
md5sum = 8644e98038ab76a8e5a499e22f92660a
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/Caddyfile.in
mode = 0644
[template-caddy-fluentd]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-caddy-fluentd.cfg.in
md5sum = ccfc6d0a4dfe0217cce5d369eea5cbd1
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
mode = 0644
......
......@@ -3,13 +3,6 @@
port 5438
</source>
#<source>
# @type dummy
# tag dummy
# dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
#</source>
# HTTP input
# http://localhost:8888/<tag>?json=<json>
#<source>
......
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import SocketServer
import time
import threading
import os
class TestServerHandler(BaseHTTPRequestHandler):
def _set_headers(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
def do_GET(self):
self._set_headers()
self.wfile.write("<html><body><h1>hi!</h1></body></html>")
def do_HEAD(self):
self._set_headers()
def do_POST(self):
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
post_data = self.rfile.read(content_length) # <--- Gets the data itself
self._set_headers()
self.wfile.write(post_data)
global posted_data
posted_data = post_data
print("post data from do_POST")
print(posted_data)
# global all_data
# all_data.append(post_data.split(" ")[1])
# global request_tag
# request_tag = find_tag(self.requestline,"=", " ")
def run():
port=9443
#server_address = ('0.0.0.0', port)
server_address = ("10.0.46.242", port)
server = HTTPServer(server_address, TestServerHandler)
thread = threading.Thread(target=server.serve_forever)
thread.start()
print("start server")
time.sleep(60)
server.shutdown()
server.server_close()
print("stop server")
if __name__ == "__main__":
run()
\ No newline at end of file
#!${buildout:directory}/bin/${eggs:interpreter}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import unittest
import requests
from StringIO import StringIO
......@@ -14,13 +10,10 @@ import os
import time
import utils
import threading
import subprocess
test_msg = "dummyInputSimpleIngest"
url = "http://" + os.environ.get('LOCAL_IPV4') + ":4443"
caddy_pidfile = os.environ.get('CADDY_DIR')
posted_data = None
all_data = []
request_tag = ""
if os.environ.get('DEBUG'):
import logging
......@@ -35,6 +28,10 @@ class FluentdPluginTestCase(utils.SlapOSInstanceTestCase):
class TestServerHandler(BaseHTTPRequestHandler):
posted_data = None
all_data = []
request_tag = ""
def _set_headers(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
......@@ -54,25 +51,21 @@ class TestServerHandler(BaseHTTPRequestHandler):
self._set_headers()
self.wfile.write(post_data)
global posted_data
posted_data = post_data
TestServerHandler.posted_data = post_data.split(" ")[1]
global all_data
all_data.append(post_data.split(" ")[1])
TestServerHandler.all_data.append(post_data.split(" ")[1])
global request_tag
request_tag = find_tag(self.requestline,"=", " ")
TestServerHandler.request_tag = find_tag(self.requestline,"=", " ")
class TestIngestion(FluentdPluginTestCase):
@classmethod
def startServer(cls):
port=9443
server_address = (os.environ.get('LOCAL_IPV4'), port)
server_address = (os.environ['LOCAL_IPV4'], port)
cls.server = HTTPServer(server_address, TestServerHandler)
cls.thread = threading.Thread(target=cls.server.serve_forever)
cls.thread.start()
time.sleep(10)
print("server start")
@classmethod
......@@ -80,98 +73,118 @@ class TestIngestion(FluentdPluginTestCase):
cls.server.shutdown()
cls.server.server_close()
print("serever shutdown")
time.sleep(10)
def test_1_get(self):
def setUp(self):
self.startServer()
time.sleep(10)
def tearDown(self):
self.stopServer()
def test_get_request_responds_with_200(self):
'''
simple get request to be sure that server is up
'''
print("############## TEST 1 ##############")
url = self.computer_partition.getConnectionParameterDict()['url']
resp = requests.get(url)
self.assertEqual(resp.status_code, 200)
print (resp.status_code)
def test_2_ingest(self):
def test_simple_ingest(self):
time.sleep(10)
print("############## TEST 2 ##############")
start_fluentd_cat(test_msg, "tag_test_2")
start_fluentd_cat(self, test_msg, "tag_test_2")
time.sleep(10)
self.assertEqual(test_msg, TestServerHandler.posted_data)
if posted_data:
self.assertEqual(test_msg, posted_data.split(" ")[1])
else:
self.assertEqual(test_msg, posted_data)
time.sleep(10)
def test_3_keepAlive_on(self):
def test_keepAlive_on(self):
print("############## TEST 3 ##############")
s = requests.session()
print("check connection type ")
print(s.headers['Connection'])
self.assertEqual('keep-alive', s.headers['Connection'])
def test_4_delay_15_mins(self):
def test_ingest_with_15mins_delay(self):
'''
sleep 15mins to test that connections doesn't break after long delay
and data is ingested correctly after the delay.
'''
print("############## TEST 4 ##############")
# sleep 15mins to test that connections doesn't break after long delay
# and data is ingested correctly after the delay.
time.sleep(900)
start_fluentd_cat("dummyInputDelay", "tag_test_4")
start_fluentd_cat(self, "dummyInputDelay", "tag_test_4")
time.sleep(15)
self.assertEqual("dummyInputDelay", posted_data.split(" ")[1])
def test_5_caddy_restart(self):
self.assertEqual("dummyInputDelay", TestServerHandler.posted_data)
def test_ingest_while_server_breakage(self):
'''
stop and then start caddy again to check that
fluentd plugin keeps message in a local buffer
and correctly sends them when caddy is back online
'''
print("############## TEST 5 ##############")
with open(caddy_pidfile) as f:
caddy_pid = f.readline()
time.sleep(10)
start_fluentd_cat("dummyInputCaddyRestart1", "tag_test_5_1")
start_fluentd_cat(self, "dummyInputCaddyRestart1", "tag_test_5_1")
time.sleep(10)
kill_caddy(caddy_pid)
time.sleep(10)
start_fluentd_cat("dummyInputCaddyRestart2 ", "tag_test_5_2")
start_fluentd_cat(self, "dummyInputCaddyRestart2 ", "tag_test_5_2")
time.sleep(10)
start_fluentd_cat("dummyInputCaddyRestart3 ", "tag_test_5_3")
start_fluentd_cat(self, "dummyInputCaddyRestart3 ", "tag_test_5_3")
time.sleep(10)
start_fluentd_cat("dummyInputCaddyRestart4 ", "tag_test_5_4")
start_fluentd_cat(self, "dummyInputCaddyRestart4 ", "tag_test_5_4")
time.sleep(130)
start_caddy(caddy_pid)
time.sleep(15)
self.assertTrue("dummyInputCaddyRestart1" in all_data)
self.assertTrue("dummyInputCaddyRestart2" in all_data)
self.assertTrue("dummyInputCaddyRestart3" in all_data)
self.assertTrue("dummyInputCaddyRestart4" in all_data)
self.assertTrue("dummyInputCaddyRestart1" in TestServerHandler.all_data)
self.assertTrue("dummyInputCaddyRestart2" in TestServerHandler.all_data)
self.assertTrue("dummyInputCaddyRestart3" in TestServerHandler.all_data)
self.assertTrue("dummyInputCaddyRestart4" in TestServerHandler.all_data)
def test_6_check_diff_tags(self):
def test_ingest_with_diff_tags(self):
'''
ingest data with different tags
'''
print("############## TEST 6 ##############")
start_fluentd_cat("dummyInputTags_6_1", "tag_Test_6_1")
start_fluentd_cat(self, "dummyInputTags_6_1", "tag_Test_6_1")
time.sleep(10)
self.assertEqual("tag_Test_6_1", request_tag)
start_fluentd_cat("dummyInputTags_6_2", "tag_Test_6_2")
time.sleep(2)
self.assertEqual("tag_Test_6_2", request_tag)
self.assertEqual("tag_Test_6_1", TestServerHandler.request_tag)
start_fluentd_cat("dummyInputTags_6_3", "tag_Test_6_3")
time.sleep(2)
self.assertEqual("tag_Test_6_3", request_tag)
self.stopServer()
start_fluentd_cat(self, "dummyInputTags_6_2", "tag_Test_6_2")
time.sleep(10)
self.assertEqual("tag_Test_6_2", TestServerHandler.request_tag)
def start_fluentd_cat(test_msg, tag):
start_fluentd_cat(self, "dummyInputTags_6_3", "tag_Test_6_3")
time.sleep(10)
self.assertEqual("tag_Test_6_3", TestServerHandler.request_tag)
def start_fluentd_cat(self, test_msg, tag):
"""Feeds `test_msg` with `tag` to fluentd.
"""
fluent_service = os.environ.get('FLUENT_SERVICE')
os.environ["GEM_PATH"] = fluent_service + "/lib/ruby/gems/1.8/"
fluentd_cat_exec_comand = fluent_service + '/bin/fluent-cat --none ' + tag + " -p 5438 "
os.system("echo + " + test_msg + " | " + fluentd_cat_exec_comand)
proc = subprocess.Popen(
[fluent_service + '/bin/fluent-cat',
'--none',
tag,
'-p',
'5438',
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={"GEM_PATH": fluent_service + "/lib/ruby/gems/1.8/" })
stdout, stderr = proc.communicate("+ " + test_msg)
self.assertEqual(0, proc.wait())
self.assertEqual('', stdout)
self.assertEqual('', stderr)
def kill_caddy(caddy_pid):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment