grafana: new software

software/grafana/README.md

+# grafana / telegraf / influxdb
+
+ 
+## Custom telegraf plugins
+
+
+See https://github.com/influxdata/telegraf to learn about plugins.
+
+Useful plugins in this context are probably
+[exec](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/exec)
+or
+[httpjson](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/httpjson).
+
+Telegraf will save in the `telegraf` database from the embedded influxdb server.
+
+
+## Grafana
+
+You'll have to add yourself the influxdb data source in grafana, using the
+parameters published by the slapos instance.
+
+http://docs.grafana.org/features/datasources/influxdb/
+
+When adding datasource, use *proxy* option, otherwise Grafana makes your
+browser query influxdb directly, which also uses a self signed certificate.
+One workaround is to configure your browser to also accept influxdb certificate
+before using grafana, but using proxy seems easier.
+
+## TODO
+
+* make a configuration option to send emails (to invite users)
+* influxdb and telegraf runs with very low priority, this could become an option
+* make one partition for each service and use switch software type
+* make it easier to add custom configuration (how ?)

software/grafana/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-profile]
+filename = instance.cfg.in
+md5sum = f9b717fa7c761686134be24c42cb9b77
+
+[influxdb-config-file]
+filename = influxdb-config-file.cfg.in
+md5sum = 3867923a2798517bc72c265066317843
+
+[telegraf-config-file]
+filename = telegraf-config-file.cfg.in
+md5sum = a1a9c22c2a7829c66a49fc2504604d21
+
+[grafana-config-file]
+filename = grafana-config-file.cfg.in
+md5sum = ce89a2a93e3b55adde9bedbea13208ac

software/grafana/influxdb-config-file.cfg.in

+reporting-disabled = false
+bind-address = "[{{ influxdb['host'] }}]:{{ influxdb['port'] }}"
+
+[meta]
+  dir = "{{ influxdb['data-dir'] }}/meta"
+  retention-autocreate = true
+  logging-enabled = true
+
+[data]
+  dir = "{{ influxdb['data-dir'] }}/data"
+  index-version = "inmem"
+  wal-dir = "{{ influxdb['data-dir'] }}/wal"
+  wal-fsync-delay = "0s"
+  query-log-enabled = true
+  cache-max-memory-size = 1073741824
+  cache-snapshot-memory-size = 26214400
+  cache-snapshot-write-cold-duration = "10m0s"
+  compact-full-write-cold-duration = "4h0m0s"
+  max-series-per-database = 1000000
+  max-values-per-tag = 100000
+  max-concurrent-compactions = 0
+  trace-logging-enabled = false
+
+[coordinator]
+  write-timeout = "10s"
+  max-concurrent-queries = 0
+  query-timeout = "0s"
+  log-queries-after = "0s"
+  max-select-point = 0
+  max-select-series = 0
+  max-select-buckets = 0
+
+[retention]
+  enabled = true
+  check-interval = "30m0s"
+
+[shard-precreation]
+  enabled = true
+  check-interval = "10m0s"
+  advance-period = "30m0s"
+
+[monitor]
+  store-enabled = true
+  store-database = "_internal"
+  store-interval = "10s"
+
+[subscriber]
+  enabled = true
+  http-timeout = "30s"
+  insecure-skip-verify = false
+  ca-certs = ""
+  write-concurrency = 40
+  write-buffer-size = 1000
+
+[http]
+  enabled = true
+  bind-address = "[{{ influxdb['host'] }}]:{{ influxdb['http-port'] }}"
+  auth-enabled = true
+  log-enabled = true
+  write-tracing = false
+  pprof-enabled = true
+  https-enabled = true
+  https-certificate = "{{ influxdb['ssl-cert-file'] }}"
+  https-private-key = "{{ influxdb['ssl-key-file'] }}"
+  max-row-limit = 0
+  max-connection-limit = 0
+  shared-secret = ""
+  realm = "InfluxDB"
+  unix-socket-enabled = true
+  bind-socket = "{{ influxdb['unix-socket'] }}"
+  max-body-size = 25000000
+
+[ifql]
+  enabled = false
+  log-enabled = true
+  bind-address = ":8082"
+
+[[graphite]]
+  enabled = false
+  bind-address = ":2003"
+  database = "graphite"
+  retention-policy = ""
+  protocol = "tcp"
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "1s"
+  consistency-level = "one"
+  separator = "."
+  udp-read-buffer = 0
+
+[[collectd]]
+  enabled = false
+  bind-address = ":25826"
+  database = "collectd"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "10s"
+  read-buffer = 0
+  typesdb = "/usr/share/collectd/types.db"
+  security-level = "none"
+  auth-file = "/etc/collectd/auth_file"
+  parse-multivalue-plugin = "split"
+
+[[opentsdb]]
+  enabled = false
+  bind-address = ":4242"
+  database = "opentsdb"
+  retention-policy = ""
+  consistency-level = "one"
+  tls-enabled = false
+  certificate = "/etc/ssl/influxdb.pem"
+  batch-size = 1000
+  batch-pending = 5
+  batch-timeout = "1s"
+  log-point-errors = true
+
+[[udp]]
+  enabled = false
+  bind-address = ":8089"
+  database = "udp"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  read-buffer = 0
+  batch-timeout = "1s"
+  precision = ""
+
+[continuous_queries]
+  log-enabled = true
+  enabled = true
+  query-stats-enabled = false
+  run-interval = "1s"
+

software/grafana/instance.cfg.in

+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+srv = ${:home}/srv
+service = ${:etc}/service
+promise = ${:etc}/promise
+influxdb-data-dir = ${:srv}/influxdb
+grafana-dir = ${:srv}/grafana
+grafana-data-dir = ${:grafana-dir}/data
+grafana-logs-dir = ${:var}/log
+grafana-plugins-dir = ${:grafana-dir}/plugins
+grafana-provisioning-config-dir = ${:grafana-dir}/provisioning-config
+grafana-provisioning-datasources = ${:grafana-provisioning-config-dir}/datasources
+grafana-provisioning-dashboards = ${:grafana-provisioning-config-dir}/dashboards
+telegraf-dir = ${:srv}/telegraf
+telegraf-extra-config-dir = ${:telegraf-dir}/extra-config
+
+
+# macros
+[generate-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${:key-file} ]
+  then
+    {{ openssl_bin }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
+      -newkey rsa:1024 -keyout ${:key-file} \
+      -out ${:cert-file}
+  fi
+update-command = ${:command}
+key-file = ${directory:etc}/${:_buildout_section_name_}.key
+cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
+common-name = ${:_buildout_section_name_}
+
+[config-file]
+recipe = slapos.recipe.template:jinja2
+template = {{ buildout['parts-directory'] }}/${:_buildout_section_name_}/${:_buildout_section_name_}.cfg.in
+rendered = ${directory:etc}/${:_buildout_section_name_}.cfg
+mode = 0644
+extensions = jinja2.ext.do
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+
+
+[influxdb]
+ipv6 = ${instance-parameter:ipv6-random}
+host = ${:ipv6}
+port = 8088
+http-port = 8086
+url = https://[${:host}]:${:http-port}
+data-dir = ${directory:influxdb-data-dir}
+auth-username = ${influxdb-password:username}
+auth-password = ${influxdb-password:passwd}
+unix-socket = ${directory:var}/influxdb.socket
+ssl-cert-file = ${influxdb-certificate:cert-file}
+ssl-key-file = ${influxdb-certificate:key-file}
+database = telegraf
+
+recipe = slapos.cookbook:wrapper
+command-line =
+   nice -19 chrt --idle 0 ionice -c3 {{ influxd_bin }} -config ${influxdb-config-file:rendered}
+wrapper-path = ${directory:service}/influxdb
+
+[influxdb-config-file]
+<= config-file
+context =
+  section influxdb influxdb
+
+[influxdb-password]
+recipe = slapos.cookbook:generate.password
+username = influxdb
+
+[influxdb-certificate]
+<= generate-certificate
+
+[influxdb-listen-promise]
+<= check-port-listening-promise
+hostname= ${influxdb:ipv6}
+port = ${influxdb:port}
+
+[influxdb-password-promise]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}
+
+
+
+[grafana]
+ipv6 = ${instance-parameter:ipv6-random}
+port = 8080
+url = https://[${:ipv6}]:${:port}
+
+data-dir = ${directory:grafana-data-dir}
+logs-dir = ${directory:grafana-logs-dir}
+plugins-dir = ${directory:grafana-plugins-dir}
+provisioning-config-dir = ${directory:grafana-provisioning-config-dir}
+admin-user = ${grafana-password:username}
+admin-password = ${grafana-password:passwd}
+secret-key = ${grafana-secret-key:passwd}
+ssl-key-file = ${grafana-certificate:key-file}
+ssl-cert-file = ${grafana-certificate:cert-file}
+
+recipe = slapos.cookbook:wrapper
+command-line =
+    {{ grafana_bin }} -config ${grafana-config-file:rendered} -homepath {{ grafana_homepath }}
+wrapper-path = ${directory:service}/grafana
+
+[grafana-certificate]
+<= generate-certificate
+
+[grafana-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+
+[grafana-secret-key]
+recipe = slapos.cookbook:generate.password
+
+[grafana-config-file]
+<= config-file
+context =
+  section grafana grafana
+
+[grafana-listen-promise]
+<= check-port-listening-promise
+hostname= ${grafana:ipv6}
+port = ${grafana:port}
+
+
+
+[telegraf]
+recipe = slapos.cookbook:wrapper
+extra-config-dir = ${directory:telegraf-extra-config-dir}
+command-line =
+   nice -19 chrt --idle 0 ionice -c3 {{ telegraf_bin }} --config ${telegraf-config-file:rendered} --config-directory ${:extra-config-dir}
+wrapper-path = ${directory:service}/telegraf
+
+[telegraf-config-file]
+<= config-file
+context =
+  section influxdb influxdb
+  section telegraf telegraf
+
+
+
+[promises]
+recipe =
+instance-promises =
+  ${influxdb-listen-promise:path}
+  ${influxdb-password-promise:wrapper-path}
+  ${grafana-listen-promise:path}
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+
+influxdb-url = ${influxdb:url}
+influxdb-database = ${influxdb:database}
+influxdb-username = ${influxdb:auth-username}
+influxdb-password = ${influxdb:auth-password}
+telegraf-extra-config-dir = ${telegraf:extra-config-dir}
+url = ${grafana:url}
+grafana-username = ${grafana:admin-user}
+grafana-password = ${grafana:admin-password}
+

software/grafana/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../stack/nodejs.cfg
+  ../../component/make/buildout.cfg
+  ../../component/golang/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  buildout.hash.cfg
+  gowork.cfg
+
+versions = versions
+parts =
+  slapos-cookbook
+  instance-profile
+  gowork
+  influxdb-config-file
+  telegraf-config-file
+  grafana-config-file
+
+
+[nodejs]
+<= nodejs-8.6.0
+
+[yarn]
+# this could become a component, but it needs to be invoked from nodejs explicitly,
+# otherwise it uses system's nodejs
+recipe = slapos.recipe.build:download-unpacked
+url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
+md5sum = db82fa09c996e9318f2f1d2ab99228f9
+
+
+[gowork]
+# All the softwares installed in the go work have "non standard" installation
+# methods, so we install them in specific parts with custom commands.
+# They will be installed because they are dependencies of ${gowork.goinstall}
+install =
+
+telegraf-bin = ${:bin}/telegraf
+influx-bin = ${:bin}/influx
+influxd-bin = ${:bin}/influxd
+grafana-bin = ${:bin}/grafana-server
+grafana-homepath = ${go_github.com_grafana_grafana:location}
+
+[gowork.goinstall]
+command = :
+depends = 
+   ${influxdb-install:recipe}
+   ${telegraf-install:recipe}
+   ${grafana-install:recipe}
+
+[influxdb-install]
+<= gowork.goinstall
+command = bash -c ". ${gowork:env.sh} && \
+   cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
+   go install ./cmd/..."
+update-command =
+
+[telegraf-install]
+<= gowork.goinstall
+command = bash -c ". ${gowork:env.sh} && \
+   cd ${gowork:directory}/src/github.com/influxdata/telegraf && \
+   ${make:location}/bin/make &&
+   cp telegraf ${gowork:bin}"
+update-command =
+
+[grafana-install]
+<= gowork.goinstall
+# yarn and go run build.go needs our nodejs in $PATH
+command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
+   . ${gowork:env.sh} && \
+   cd ${gowork:directory}/src/github.com/grafana/grafana && \
+   ${gowork:golang}/bin/go run build.go setup && \
+   ${gowork:golang}/bin/go run build.go build && \
+   ${yarn:location}/bin/yarn install --pure-lockfile && \
+   ${nodejs:location}/bin/npm run build"
+update-command =
+
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[influxdb-config-file]
+<= download-file-base
+
+[telegraf-config-file]
+<= download-file-base
+
+[grafana-config-file]
+<= download-file-base
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/instance.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  section buildout buildout
+  key openssl_bin openssl-output:openssl
+  key telegraf_bin gowork:telegraf-bin
+  key influxd_bin gowork:influxd-bin
+  key influx_bin gowork:influx-bin
+  key grafana_bin gowork:grafana-bin
+  key grafana_homepath gowork:grafana-homepath
+
+
+[versions]
+slapos.recipe.template = 4.2
+inotifyx = 0.2.2
+

software/grafana/telegraf-config-file.cfg.in

+# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared plugins.
+
+# Even if a plugin has no configuration, it must be declared in here
+# to be active. Declaring a plugin means just specifying the name
+# as a section with no variables. To deactivate a plugin, comment
+# out the name and any variables.
+
+# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
+# file would generate.
+
+# One rule that plugins conform to is wherever a connection string
+# can be passed, the values '' and 'localhost' are treated specially.
+# They indicate to the plugin to use their own builtin configuration to
+# connect to the local system.
+
+# NOTE: The configuration has a few required parameters. They are marked
+# with 'required'. Be sure to edit those to make this configuration work.
+
+# Tags can also be specified via a normal map, but only one form at a time:
+[tags]
+  # dc = "us-east-1"
+
+# Configuration for telegraf agent
+[agent]
+  # Default data collection interval for all plugins
+  interval = "10s"
+  # Rounds collection interval to 'interval'
+  # ie, if interval="10s" then always collect on :00, :10, :20, etc.
+  round_interval = true
+
+  # Default data flushing interval for all outputs. You should not set this below
+  # interval. Maximum flush_interval will be flush_interval + flush_jitter
+  flush_interval = "10s"
+  # Jitter the flush interval by a random amount. This is primarily to avoid
+  # large write spikes for users running a large number of telegraf instances.
+  # ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
+
+  # Run telegraf in debug mode
+  debug = false
+  # Override default hostname, if empty use os.Hostname()
+  hostname = ""
+
+
+###############################################################################
+#                                  OUTPUTS                                    #
+###############################################################################
+
+[outputs]
+
+# Configuration for influxdb server to send metrics to
+[outputs.influxdb]
+  # The full HTTP or UDP endpoint URL for your InfluxDB instance
+  # Multiple urls can be specified for InfluxDB cluster support.
+  # urls = ["udp://localhost:8089"] # UDP endpoint example
+  # XXX XXX XXX
+  #urls = ["http://localhost:8086"] # required
+  urls = ["{{ influxdb['url'] }}"]
+  insecure_skip_verify = true # because we are using a self signed certificate
+  # The target database for metrics (telegraf will create it if not exists)
+  database = "{{ influxdb['database'] }}" # required
+  # Precision of writes, valid values are n, u, ms, s, m, and h
+  # note: using second precision greatly helps InfluxDB compression
+  precision = "s"
+
+  # Connection timeout (for the connection with InfluxDB), formatted as a string.
+  # If not provided, will default to 0 (no timeout)
+  # timeout = "5s"
+  username = "{{ influxdb['auth-username'] }}"
+  password = "{{ influxdb['auth-password'] }}"
+  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
+  # user_agent = "telegraf"
+  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
+  # udp_payload = 512
+
+
+###############################################################################
+#                                  PLUGINS                                    #
+###############################################################################
+
+# Read metrics about cpu usage
+[cpu]
+  # Whether to report per-cpu stats or not
+  percpu = true
+  # Whether to report total system cpu stats or not
+  totalcpu = true
+  # Comment this line if you want the raw CPU time metrics
+  drop = ["cpu_time"]
+
+# Read metrics about memory usage
+[mem]
+  # no configuration
+
+[disk]
+
+[io]
+
+[system]
+
+
+###############################################################################
+#                           ERP5 - PLUGINS                                    #
+###############################################################################
+#
+# Left here as example, don't edit this file directly, but place your config
+# files in {{ telegraf['extra-config-dir'] }}
+# 
+
+#[mysql]
+#  servers = ["root@unix(/srv/slapgrid/slappart12/srv/runner/instance/slappart1/var/run/mariadb.sock)/erp5"]
+
+#[memcached]
+#  # XXX kumofs does not support memcached's stat command
+#  servers = ["10.0.248.233:2013", "10.0.248.233:2003"]
+
+#[haproxy]
+#  servers = ["http://10.0.121.162:2150/haproxy", "http://10.0.121.162:2152/haproxy"]
+
+#[[inputs.exec]]
+#  commands = ["/srv/slapgrid/slappart0/bin/slapsensor  /srv/slapgrid/slappart0/srv/runner/instance/etc/supervisord.conf"]
+#  name_suffix = "_slapos"
+#  interval = "5s"
+
+
+
+###############################################################################
+#                              SERVICE PLUGINS                                #
+###############################################################################