monitor: remove fluentd

software/monitor/fluentd-agent.conf.jinja2.in

-# DESTINATION
-<match td.*.*>
-  type tdlog
-  apikey YOUR_API_KEY
-
-  auto_create_table
-  buffer_type file
-  buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
-
-  <secondary>
-    type file
-    path {{ fluentd_log_directory }}/failed_records
-  </secondary>
-</match>
-
-{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '')  -%}
-{% set wendelin_password = slapparameter_dict.get('wendelin-password', '')  -%}
-{% set wendelin_user = slapparameter_dict.get('wendelin-user', '')  -%}
-{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN')  -%}
-{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
-<source>
-  type syslog
-  port 42185
-  bind 127.0.0.1
-  tag {{ wendelin_uid }}.system
-</source>
-
-<match {{ wendelin_uid }}.system.**>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_syslog
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-<source>
-  type tail
-  path {{ fluentd_log_directory }}/consumption/*/dump_user.csv
-  pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos
-  tag slapos.monitor.usage.user.{{ wendelin_uid }}
-  format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
-  read_from_head true
-</source>
-
-<match slapos.monitor.usage.user.{{ wendelin_uid }}>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_user
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-<source>
-  type tail
-  path {{ fluentd_log_directory }}/consumption/*/dump_system.csv
-  pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos
-  tag slapos.monitor.usage.system.{{ wendelin_uid }}
-  format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
-  read_from_head true
-</source>
-
-<match slapos.monitor.usage.system.{{ wendelin_uid }}>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_system
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-
-<source>
-  type tail
-  path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv
-  pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos
-  tag slapos.monitor.usage.computer.{{ wendelin_uid }}
-  format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
-  
-  read_from_head true
-</source>
-
-<match slapos.monitor.usage.computer.{{ wendelin_uid }}>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_computer
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-<source>
-  type tail
-  path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv
-  pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos
-  tag slapos.monitor.usage.disk.{{ wendelin_uid }}
-  format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
-  read_from_head true
-</source>
-
-<match slapos.monitor.usage.disk.{{ wendelin_uid }}>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_disk
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-<source>
-  type tail
-  path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv
-  pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos
-  tag slapos.monitor.usage.temperature.{{ wendelin_uid }}
-  format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
-  read_from_head true
-</source>
-
-<match slapos.monitor.usage.temperature.{{ wendelin_uid }}>
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }}_temperature
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-
-{% endif -%}

software/monitor/fluentd-distributor-agent.conf.jinja2.in

-# DESTINATION
-<match td.*.*>
-  type tdlog
-  apikey YOUR_API_KEY
-
-  auto_create_table
-  buffer_type file
-  buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
-
-  <secondary>
-    type file
-    path {{ fluentd_log_directory }}/failed_records
-  </secondary>
-</match>
-
-## match tag=debug.** and dump to console
-<match debug.**>
-  type stdout
-</match>
-
-<source>
-  type tail
-  path {{ crawl_log_directory }}/*/*ping.log.20*
-  pos_file {{ crawl_log_directory }}/tail_in_ping.pos
-  tag slapos.monitor.networktest.ping.ipv4
-  format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
-  read_from_head true
-</source>
-
-<source>
-  type tail
-  path {{ crawl_log_directory }}/*/*ping6.log.20*
-  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
-  tag slapos.monitor.networktest.ping.ipv6
-  format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
-  read_from_head true
-</source>
-
-<match slapos.monitor.networktest.ping.*>
-  type copy
-{% for slave_instance in slave_instance_list -%}
-{% if  slave_instance.get("ping_ip_list") -%} 
-  <store>
-    type grep
-    regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
-    add_tag_prefix {{ slave_instance.get("slave_reference") }}
-  </store>
-{% endif -%}
-{% if  slave_instance.get("ping6_ip_list") -%}
-  <store>
-    type grep
-    regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
-    add_tag_prefix {{ slave_instance.get("slave_reference") }}
-  </store>
-{% endif -%}
-{% endfor -%}
-</match>
-
-{% for slave_instance in slave_instance_list -%}
-<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
-  type file
-  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
-  append true
-  include_time_key true
-  include_tag_key true
-  format json
-</match>
-<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
-  type file
-  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
-  append true
-  include_time_key true
-  include_tag_key true
-  format json
-</match>
-{% endfor -%}
-
-<source>
-  type tail
-  path {{ network_user_logs }}/*/ping*/log/*.log
-  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
-  tag slapos.wendelin.networktest
-  format json 
-  read_from_head true
-</source>
-
-{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '')  -%}
-{% set wendelin_password = slapparameter_dict.get('wendelin-password', '')  -%}
-{% set wendelin_user = slapparameter_dict.get('wendelin-user', '')  -%}
-{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
-<match slapos.wendelin.networktest>
-  
-  @type wendelin
-  @id wendelin_out
-
-  streamtool_uri {{ wendelin_streamtool_uri }} 
-  user      {{ wendelin_user }}
-  password  {{ wendelin_password }}
-
-  buffer_type       memory
-  flush_interval    20s
-</match>
-{% endif -%}
-
-## SOURCE
-<source>
-  type forward
-</source>
-
-## live debugging agent
-<source>
-  type debug_agent
-  bind 127.0.0.1
-  port 24230
-</source>
-

software/monitor/instance-monitor-distributor.cfg.jinja2

@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler
 extends = {{ instance_base_monitor }}
 parts +=
   slave-test-configuration
-  fluentd-distributor-wrapper
 {% for part in part_list %}
 {{ '  %s' % part }}
 {% endfor %}
 
-[fluentd-distributor-wrapper]
-<=fluentd-wrapper
-command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered}
-wrapper-path = ${monitor-directory:service}/fluentd-distributor
-
-[fluentd-distributor-conf-configuration]
-recipe = slapos.recipe.template:jinja2
-template = {{ fluent_distributor_conf_output }}
-rendered = ${monitor-directory:etc}/fluentd-distributor.cfg
-mode = 0744
-context =
-  key slapparameter_dict slap-configuration:configuration
-  key slave_instance_list slap-configuration:slave-instance-list
-  key fluentd_log_directory monitor-directory:fluentd-log
-  key crawl_log_directory monitor-directory:crawl-log
-  key network_user_logs monitor-directory:network-user-logs
-  key computer_id slap-configuration:computer
-

software/monitor/instance-monitor.cfg.jinja2

@@ -8,7 +8,6 @@ parts =
   symlink-re6st-logs
   symlink-collected-logs
   monitor-collect-csv-wrapper
-  fluentd-wrapper
   monitor-base
   monitor-check-memory-usage
   monitor-check-cpu-usage
@@ -24,7 +23,10 @@ offline = true
 recipe = slapos.cookbook:cron.d
 name = network-bench-test
 frequency = */10 * * * *
-command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
+# skip to not fill cron log file.
+# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
+command = true
+
 
 [symlink-re6st-logs]
 recipe = cns.recipe.symlink
@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log
 cache = ${:var}/cache
 mod-ssl = ${:cache}/httpd_mod_ssl
 system-log = ${:private}/system-log
-fluentd-log = ${:log}/fluentd
-consumption = ${:fluentd-log}/consumption
+consumption = ${:log}/consumption
 
 [monitor-httpd-configuration-file]
 context =
@@ -76,28 +77,10 @@ context =
   section slave_information slap-configuration
   key monitor_private_hash slap-configuration:private-hash  
 
-[fluentd-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
-wrapper-path = ${monitor-directory:service}/fluentd
-environment = 
-  GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
-
-[fluentd-conf-configuration]
-recipe = slapos.recipe.template:jinja2
-template = {{ fluent_conf_output }}
-rendered = ${monitor-directory:etc}/fluentd.cfg
-mode = 0744
-context =
-  key slapparameter_dict slap-configuration:configuration
-  key slave_instance_list slap-configuration:slave-instance-list
-  key fluentd_log_directory monitor-directory:fluentd-log
-  key computer_id slap-connection:computer-id
-
 [monitor-collect-csv-wrapper]
 recipe = slapos.cookbook:wrapper
 command-line = 
-  ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/
+  ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
 wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump  
 
 [monitor-check-cpu-usage]

software/monitor/instance.cfg

@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2
 template = ${template-monitor:destination}
 rendered = $${buildout:directory}/template-base-monitor.cfg
 extensions = jinja2.ext.do
-context = key buildout buildout:bin-directory
-          key develop_eggs_directory buildout:develop-eggs-directory
+context = key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
+          raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}
-          raw network_benck_cfg_output ${network-bench-cfg:output} 
-          raw fluentd_location ${fluentd:location}
-          raw fluent_conf_output ${fluentd-agent-conf:output}
+          raw network_benck_cfg_output ${network-bench-cfg:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
           raw monitor_check_system_health ${monitor-system-health:output}
 mode = 0644
@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination}
 rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
 extensions = jinja2.ext.do
 context = import json_module json
-          key buildout buildout:bin-directory
           key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
           key instance_base_monitor instance-base-monitor:rendered
           key slave_instance_list slap-configuration:slave-instance-list 
+          raw buildout_bin ${buildout:bin-directory}
           raw template_json_distributor_test ${json-test-template:destination}
-          raw fluentd_location ${fluentd:location}
-          raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output}
           raw wget_bin ${wget:location}/bin/wget
 mode = 0644
 

software/monitor/software.cfg

@@ -5,7 +5,6 @@ extends =
   ../../component/python-cryptography/buildout.cfg
   ../../component/wget/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../../component/fluentd/buildout.cfg
   ../../stack/slapos.cfg
 
 parts = 
@@ -16,30 +15,27 @@ parts =
   template
   template-monitor-distributor
   template-monitor
-  fluentd
-  fluentd-agent-conf
-  fluentd-agent-distributor-conf
   monitor-collect-csv-dump
 
 [template]
 recipe = slapos.recipe.template 
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9
+md5sum = 641c5916739f78171c616af00fe974a2
 mode = 0644
 
 [template-monitor]
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
 destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = 5def53c8faa0dfca313e53ebcc3229fa
+md5sum = 5cd3fe3edb4a7fbbc9671f051845d752
 mode = 0644
 
 [template-monitor-distributor]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
 destination = ${buildout:directory}/template-monitor-base-distributor.cfg
-md5sum = 0ab9a86df18125335ed365bb673bbcf3
+md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
 mode = 0644
 
 [json-test-template]
@@ -56,20 +52,6 @@ md5sum  = cfcbf2002b8eff5153e2bf68ed24b720
 output = ${buildout:directory}/template-network-bench-cfg.in
 mode = 0644
 
-[fluentd-agent-conf]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
-md5sum = 3ea59906937eab7aeef78f46c4994ecd
-output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
-mode = 0644
-
-[fluentd-agent-distributor-conf]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in
-md5sum = 4b81ddcbe2f16d23013caac37151d396
-output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in
-mode = 0644
-
 [monitor-collect-csv-dump]
 <= monitor-template-script
 url = ${:_profile_base_location_}/script/${:filename}
@@ -104,12 +86,6 @@ scripts =
   monitor.configwrite
   is-process-older-than-dependency-set
 
-[fluentd]
-gems += 
-  fluent-plugin-wendelin==0.1.alpha1
-  fluent-plugin-grep==0.3.4
-  fluent-plugin-bin==0.1
-
 [monitor-eggs]
 eggs += 
   slapos.cookbook