WIP grafana

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 39a1ee09ca7a12995703ff2a6a869637
+md5sum = bdc556fa76a3f5a763391797c7f3e342
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in
@@ -23,7 +23,7 @@ md5sum = a28972ced3e0f4aa776e43a9c44717c0
 
 [telegraf-config-file]
 filename = telegraf-config-file.cfg.in
-md5sum = 6de1faa34842e1eda095a51edecc2083
+md5sum = 016d0163ca3dbabe538a8feeee745c60
 
 [grafana-config-file]
 filename = grafana-config-file.cfg.in
@@ -39,8 +39,4 @@ md5sum = 5616679a9c5c2757540175ead3f5500a
 
 [loki-config-file]
 filename = loki-config-file.cfg.in
-md5sum = 19a7f5cb904b3287b0bc7cb3e8a27429
-
-[loki-nginx-config-file]
-filename = loki-nginx-config-file.cfg.in
-md5sum = b08ce1e4abb34eb79e26133459c27c3a
+md5sum = e4917bae2a07598ce5305520e9a58b6d

software/grafana/loki-config-file.cfg.in

-# insipired from
-# https://github.com/grafana/loki/blob/1489c1731277c327e3661da182bfc6c90d4559f4/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
-# and othe configuration examples with microservices, because the single binary
-# mode assumes running on 127.0.0.1, but in slapos we want to bind on partition's
-# addresses
-
 auth_enabled: false
 
-http_prefix:
-
 server:
-  http_listen_address: {{ loki['ip'] }}
-  grpc_listen_address: {{ loki['ip'] }}
+  http_listen_address: {{ loki['ipv6'] }}
+  http_listen_port: {{ loki['http-port'] }}
+  http_tls_ca_path: {{ loki['ca-file'] }}
+  http_tls_cert_path: {{ loki['cert-file'] }}
+  http_tls_key_path: {{ loki['key-file'] }}
+  http_tls_client_auth_type: RequireAndVerifyClientCert
+
+  grpc_listen_address: {{ loki['ipv4'] }}
   grpc_server_max_recv_msg_size: 1.048576e+08
   grpc_server_max_send_msg_size: 1.048576e+08
 
-# # TODO ?
-# wal:
-#   enabled: true
-#   dir: /loki/wal
 
 common:
-  compactor_address: http://{{ loki['ip'] }}:{{ loki['write-http-port'] }}
+  ring:
+    instance_addr: {{ loki['ipv4'] }}
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: {{ loki['path-prefix'] }}
 
 schema_config:
   configs:
   - from: 2020-05-15
-    store: boltdb-shipper
+    store: tsdb
     object_store: filesystem
-    schema: v11
+    schema: v13
     index:
       prefix: index_
       period: 24h
 
 storage_config:
-  boltdb_shipper:
-    active_index_directory: {{ loki['boltdb-shipper-active-index-directory'] }}
-    cache_location: {{ loki['boltdb-shipper-cache-location'] }}
   filesystem:
     directory: {{ loki['storage-filesystem-directory'] }}
 
@@ -45,42 +41,8 @@ limits_config:
   ingestion_rate_mb: 1024
   ingestion_burst_size_mb: 1024
 
-
-ingester:
-  lifecycler:
-    address: {{ loki['ip'] }}
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-
-compactor:
-    compaction_interval: 1m
-    retention_enabled: true
-    working_directory: {{ loki['compactor-working-directory'] }}
-
-frontend:
-  log_queries_longer_than: 5s
-  compress_responses: true
-  max_outstanding_per_tenant: 2048
-  tail_proxy_url: http://{{ loki['ip'] }}:{{ loki['querier-http-port']}}
-
+# https://github.com/grafana/loki/issues/5143#issuecomment-1697196679
 frontend_worker:
-  scheduler_address: {{ loki['ip'] }}:{{ loki['query-scheduler-grpc-port'] }}
-#testERP5Type
-memberlist:
-  bind_addr:
-    - {{ loki['ip'] }}
-
-  join_members:
-#  - {{ loki['ip'] }}:{{ loki['read-1-memberlist-port'] }}
-  - {{ loki['ip'] }}:{{ loki['querier-memberlist-port'] }}
-#  - {{ loki['ip'] }}:{{ loki['write-memberlist-port'] }}
-
-
-query_scheduler:
-  max_outstanding_requests_per_tenant: 1024
-
-querier:
-  query_ingesters_within: 2h
+   grpc_client_config:
+     max_send_msg_size: 268435456
 

software/grafana/loki-nginx-config-file.cfg.in

-daemon off;
-
-events {
-    worker_connections 1024;
-}
-
-error_log   /dev/stdout;
-
-http {
-  default_type application/octet-stream;
-  access_log   /dev/stdout;
-  sendfile     on;
-  tcp_nopush   on;
-
-  upstream read {
-    server {{ loki['ip'] }}:{{ loki['query-frontend-http-port'] }};
-  }
-
-  upstream write {
-    server {{ loki['ip'] }}:{{ loki['write-http-port'] }};
-  }
-
-  upstream cluster {
-    server {{ loki['ip'] }}:{{ loki['write-http-port'] }};
-    server {{ loki['ip'] }}:{{ loki['query-frontend-http-port'] }};
-    server {{ loki['ip'] }}:{{ loki['querier-http-port'] }};
-
-  }
-
-  upstream query-frontend {
-    server {{ loki['ip'] }}:{{ loki['query-frontend-http-port'] }};
-  }
-
-  server {
-    listen {{ loki['ip'] }}:{{ loki['nginx-port'] }};
-# XXX while debugging
-    listen [{{ loki['ipv6'] }}]:{{ loki['nginx-port'] }};
-
-    location / {
-        return 200 'OK';
-    }
-
-    location = /ring {
-        proxy_pass       http://cluster$request_uri;
-    }
-
-    location = /memberlist {
-        proxy_pass       http://cluster$request_uri;
-    }
-
-    location = /config {
-        proxy_pass       http://cluster$request_uri;
-    }
-
-    location = /metrics {
-        proxy_pass       http://cluster$request_uri;
-    }
-
-    location = /ready {
-        proxy_pass       http://cluster$request_uri;
-    }
-
-    location = /loki/api/v1/push {
-        proxy_pass       http://write$request_uri;
-    }
-
-    location = /loki/api/v1/tail {
-        proxy_pass       http://read$request_uri;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-    }
-
-    location ~ /loki/api/.* {
-        proxy_pass       http://query-frontend$request_uri;
-    }
-  }
-}

software/grafana/software.cfg

 [buildout]
 extends =
   ../../stack/slapos.cfg
+  ../../stack/caucase/buildout.cfg
   ../../stack/nodejs.cfg
   ../../component/make/buildout.cfg
   ../../component/golang/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
-  ../../component/nginx/buildout.cfg
+  ../../component/systemd/buildout.cfg
   buildout.hash.cfg
 
 parts =
@@ -20,16 +21,7 @@ parts =
   grafana-provisioning-datasources-config-file
   grafana-provisioning-dashboards-config-file
   loki-config-file
-  loki-nginx-config-file
 
-; [nodejs]
-; <= nodejs-16.19.0
-
-
-[gowork]
-# XXX speed up development cycle by not rebuilding workspace on every software run
-# XXX does not work ?
-update-command =
 
 [go_github.com_grafana_grafana]
 <= go-git-package
@@ -41,7 +33,7 @@ revision      = v10.1.2-0-g8e428858dd
 <= go-git-package
 go.importpath = github.com/grafana/loki
 repository    = https://github.com/grafana/loki
-revision      = v2.9.1-0-gd9d5ed4a1
+revision      = v3.0.0-0-gb4f7181c7
 
 [go_github.com_influxdata_influxdb]
 <= go-git-package
@@ -59,7 +51,7 @@ revision      = v1.28.1-0-g3ea9ffbe2
 <= go-git-package
 go.importpath = github.com/perrinjerome/telegraf-input-slapos
 repository    = https://github.com/perrinjerome/telegraf-input-slapos
-revision      = v0.0.1-0-gf8981f3
+revision      = v0.0.2-0-gd4c5221
 
 [go_github.com_prometheus_prometheus]
 <= go-git-package
@@ -84,15 +76,18 @@ install =
   ${go_github.com_perrinjerome_slapos_telegraf_input:location}:./...
   ${go_github.com_prometheus_prometheus:location}:./cmd/...
 
-# disable cgo, to prevent loki/promtail from using go-systemd
 environment =
-  CGO_ENABLED = 0
-
+  CGO_ENABLED=1
+  CGO_CFLAGS=-I${systemd:location}/include
+buildflags =
+  -tags promtail_journal_enabled
+cpkgpath =
+  ${systemd:location}
 telegraf-bin = ${:bin}/telegraf
 telegraf-input-slapos-bin = ${:bin}/telegraf-input-slapos
 influx-bin = ${:bin}/influx
 influxd-bin = ${:bin}/influxd
-grafana-bin = ${:bin}/grafana-server
+grafana-bin = ${grafana:binpath}/grafana
 grafana-homepath = ${grafana:homepath}
 loki-bin = ${:bin}/loki
 promtail-bin = ${:bin}/promtail
@@ -105,8 +100,6 @@ command = bash -ce "
   . ${gowork:env.sh} && \
   go install github.com/google/wire/cmd/wire@v0.5.0 && \
   wire gen -tags oss ./pkg/server ./pkg/cmd/grafana-cli/runner && \
-  # Unlike loki, grafana _needs_ CGO_ENABLED, so we override here
-  export CGO_ENABLED=1 && \
   go run build.go setup && \
   go run build.go build && \
   export NODE_OPTIONS=--max_old_space_size=8192 && \
@@ -119,6 +112,8 @@ command = bash -ce "
   rm -rf ${buildout:directory}/.cache/yarn/
   "
 homepath = ${go_github.com_grafana_grafana:location}
+# XXX "linux-amd64" is not portable here
+binpath = ${go_github.com_grafana_grafana:location}/bin/linux-amd64
 stop-on-error = true
 
 [download-file-base]
@@ -143,9 +138,6 @@ url = ${:_profile_base_location_}/${:filename}
 [loki-config-file]
 <= download-file-base
 
-[loki-nginx-config-file]
-<= download-file-base
-
 [instance-eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -167,13 +159,14 @@ context =
   key grafana_bin gowork:grafana-bin
   key grafana_homepath gowork:grafana-homepath
   key loki_bin gowork:loki-bin
-  raw nginx_bin ${nginx:location}/sbin/nginx
   key promtail_bin gowork:promtail-bin
   key curl_bin :curl-bin
   key dash_bin :dash-bin
 curl-bin = ${curl:location}/bin/curl
 dash-bin = ${dash:location}/bin/dash
 depends = ${instance-eggs:eggs}
+import-list =
+  file caucase caucase-jinja2-library:target
 
 [versions]
 inotifyx = 0.2.2

software/grafana/telegraf-config-file.cfg.in

@@ -21,7 +21,7 @@
 
 # Tags can also be specified via a normal map, but only one form at a time:
 [tags]
-  # dc = "us-east-1"
+  computer_id = "{{ slap_configuration['computer'] }}"
 
 # Configuration for telegraf agent
 [agent]
@@ -42,8 +42,7 @@
   # Run telegraf in debug mode
   debug = false
   # Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
+  hostname =
 
 ###############################################################################
 #                                  OUTPUTS                                    #
@@ -97,6 +96,7 @@
 
 [system]
 
+# TODO: generate this full config file in toml
 {{ extra['extra-config'] }}
 
 ###############################################################################