grafana split

software/grafana/buildout.hash.cfg

@@ -15,7 +15,16 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = e4d5ac3e6ad239d3bf48c2b3172919b5
+md5sum = 32c772c593d2c3c38c26186b91b78cf8
+
+[instance-default]
+filename = instance-default.cfg.in
+md5sum = 5a9650b8654b2aeaeab076b08b248b70
+
+[instance-agent]
+filename = instance-agent.cfg.in
+md5sum = edb4eeb900ad13a3b3a6e174c1ea533b
+
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in
@@ -23,9 +32,10 @@ md5sum = a28972ced3e0f4aa776e43a9c44717c0
 
 [grafana-config-file]
 filename = grafana-config-file.cfg.in
-md5sum = 83a8445858eab21a12f1769c23424bea
+md5sum = 2b75d6b1984d9d154303ec773aa88474
 
 [grafana-provisioning-dashboards-config-file]
 filename = grafana-provisioning-dashboards-config-file.cfg.in
 md5sum = 5616679a9c5c2757540175ead3f5500a
 
+

software/grafana/grafana-config-file.cfg.in

@@ -334,20 +334,21 @@ allow_sign_up = true
 
 #################################### SMTP / Emailing #####################
 [smtp]
+{% set email = slapparameter_dict.get('email', {}) %}
 #enabled = false
-enabled = {{ slapparameter_dict.get('smtp-server') and 'true' or 'false' }}
+enabled = {{ email.get('smtp-server') and 'true' or 'false' }}
 #host = locahost:25
-host = {{ slapparameter_dict.get('smtp-server', '') }}
+host = {{ email.get('smtp-server', '') }}
 #user =
-user = {{ slapparameter_dict.get('smtp-username', '') }}
+user = {{ email.get('smtp-username', '') }}
 # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
 #password =
-password = {{ slapparameter_dict.get('smtp-password', '') and '"""%s"""' % slapparameter_dict['smtp-password'] or ""}}
+password = {{ email.get('smtp-password', '') and '"""%s"""' % email['smtp-password'] or ""}}
 cert_file =
 key_file =
-skip_verify = {{ slapparameter_dict.get('smtp-verify-ssl') and 'true' or 'false' }}
-from_address = {{ slapparameter_dict.get('email-from-address', '') }}
-from_name = {{ slapparameter_dict.get('email-from-name', 'Grafana') }}
+skip_verify = {{ email.get('smtp-verify-ssl') and 'false' or 'true' }}
+from_address = {{ email.get('email-from-address', '') }}
+from_name = {{ email.get('email-from-name', 'Grafana') }}
 ehlo_identity =
 
 [emails]

software/grafana/instance-agent-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema",
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
   "description": "Parameters to instantiate an agent collecting logs and metrics",
   "type": "object",
   "additionalProperties": false,
+  "unevaluatedProperties": false,
   "$defs": {
     "type": {
       "description": "Type of the application. With `SlapOS` type, some metrics are collected from supervisor and from some known partition types (for example: ERP5's mariadb or ERP5's zopes). With `system` type, only log files are ingested.",
@@ -36,7 +37,7 @@
       "description": "Static tags for this partition",
       "examples": [
         {
-          "region": "eu",
+          "service-level": "production",
           "data-center": "abc123"
         }
       ]
@@ -76,7 +77,11 @@
               },
               "instance-root": {
                 "description": "Directory containing SlapOS partitions.",
-                "type": "string"
+                "type": "string",
+                "examples": [
+                  "/srv/slapgrid/",
+                  "/srv/slapgrid/slappart30/srv/runner/instance/"
+                ]
               },
               "partitions": {
                 "description": "SlapOS partitions to monitor",
@@ -87,7 +92,7 @@
                     "name",
                     "reference"
                   ],
-                  "additionalProperties": false,
+                  "unevaluatedProperties": false,
                   "properties": {
                     "name": {
                       "type": "string",
@@ -120,13 +125,39 @@
                       "default": "default"
                     },
                     "log-file-patterns": {
-                      "$refs": "#/$defs/log-file-patterns",
+                      "$ref": "#/$defs/log-file-patterns",
                       "description": "Glob pattern for log files to watch. This mostly makes sense for `default` partition type"
                     },
                     "static-tags": {
-                      "$refs": "#/$defs/static-tags"
+                      "$ref": "#/$defs/static-tags"
                     }
                   },
+                  "allOf": [
+                    {
+                      "if": {
+                        "properties": {
+                          "type": {
+                            "enum": [
+                              "mariadb",
+                              "erp5/mariadb"
+                            ]
+                          }
+                        }
+                      },
+                      "then": {
+                        "properties": {
+                          "dbname": {
+                            "type": "string",
+                            "description": "Database name"
+                          },
+                          "username": {
+                            "type": "string",
+                            "description": "Username to connect to database"
+                          }
+                        }
+                      }
+                    }
+                  ],
                   "examples": [
                     {
                       "name": "zope-backoffice",
@@ -187,10 +218,10 @@
                       ]
                     },
                     "log-file-patterns": {
-                      "$refs": "#/$defs/log-file-patterns"
+                      "$ref": "#/$defs/log-file-patterns"
                     },
                     "static-tags": {
-                      "$refs": "#/$defs/static-tags"
+                      "$ref": "#/$defs/static-tags"
                     }
                   },
                   "examples": [

software/grafana/instance-agent-output-schema.json

@@ -6,6 +6,15 @@
     "telegraf-extra-config-dir": {
       "description": "Directory in telegraf partition where extra configuration file will be loaded. These files must match *.conf pattern",
       "type": "string"
+    },
+    "promtail-url": {
+      "description": "URL of embedded server from promtail",
+      "format": "uri",
+      "type": "string"
+    },
+    "facl-script": {
+      "description": "Path of a generated script to set ACL for the agent to access files and sockets. This might be needed depending on how slapos partitions were formatted",
+      "type": "string"
     }
   },
   "type": "object"

software/grafana/instance-default-input-schema.json

@@ -4,35 +4,66 @@
   "type": "object",
   "additionalProperties": false,
   "properties": {
-    "smtp-server": {
-      "description": "SMTP server used by Grafana to send emails (in host:port format). Leaving this empty will disable email sending.",
-      "type": "string"
-    },
-    "smtp-username": {
-      "description": "Username to connect to SMTP server",
-      "type": "string"
-    },
-    "smtp-password": {
-      "description": "Password to connect to SMTP server",
-      "type": "string"
-    },
-    "smtp-verify-ssl": {
-      "description": "Verify SSL certificate of SMTP server",
-      "type": "boolean"
-    },
-    "email-from-address": {
-      "description": "Email address used in From: header of emails",
-      "type": "string"
+    "email": {
+      "type": "object",
+      "description": "Email configuration",
+      "additionalProperties": false,
+      "properties": {
+        "smtp-server": {
+          "description": "SMTP server used by Grafana to send emails (in host:port format). Leaving this empty will disable email sending.",
+          "type": "string"
+        },
+        "smtp-username": {
+          "description": "Username to connect to SMTP server",
+          "type": "string"
+        },
+        "smtp-password": {
+          "description": "Password to connect to SMTP server",
+          "type": "string"
+        },
+        "smtp-verify-ssl": {
+          "description": "Verify certificate of SMTP server",
+          "type": "boolean",
+          "default": true
+        },
+        "email-from-address": {
+          "description": "Email address used in `From:` header of emails",
+          "type": "string"
+        },
+        "email-from-name": {
+          "description": "Name used in `From:` header of emails",
+          "default": "Grafana",
+          "type": "string"
+        }
+      }
     },
-    "email-from-name": {
-      "description": "Name used in From: header of emails",
-      "default": "Grafana",
-      "type": "string"
+    "frontend": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "custom-domain": {
+          "description": "Custom domain to use when requesting a rapid-cdn frontend",
+          "type": "string",
+          "format": "hostname"
+        }
+      }
     },
-    "caucase-url": {
-      "description": "URL of a caucase instance to manage all server and clients certificates",
-      "type": "string",
-      "format": "uri"
+    "caucase": {
+      "type": "object",
+      "description": "Caucase configuration. To connect external agents, it's required to approve their client certificates, either using an external caucase referenced as `external-caucase-url` or registering a user with `user-auto-approve-count`",
+      "additionalProperties": false,
+      "properties": {
+        "external-caucase-url": {
+          "description": "URL of a caucase instance to manage all server and clients certificates, to use instead of embedding caucase",
+          "type": "string",
+          "format": "uri"
+        },
+        "user-auto-approve-count": {
+          "description": "Number of users to automatically approve in the embedded caucase",
+          "type": "integer",
+          "default": 0
+        }
+      }
     },
     "influxdb": {
       "description": "Fine tuning influxdb parameters",

software/grafana/instance-default-output-schema.json

 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Values returned by Grafana instantiation",
-  "additionalProperties": false,
   "properties": {
     "url": {
       "description": "Shared frontend for this Grafana instance",
@@ -47,6 +46,15 @@
       "description": "URL caucase service used by Loki",
       "format": "uri",
       "type": "string"
+    },
+    "agent-promtail-url": {
+      "description": "URL of embedded server from promtail",
+      "format": "uri",
+      "type": "string"
+    },
+    "agent-facl-script": {
+      "description": "Path of a generated script to set ACL for the agent to access files and sockets. This might be needed depending on how slapos partitions were formatted",
+      "type": "string"
     }
   },
   "type": "object"

software/grafana/software.cfg

@@ -129,15 +129,18 @@ url = ${:_profile_base_location_}/${:filename}
 [grafana-provisioning-dashboards-config-file]
 <= download-file-base
 
-[loki-config-file]
-<= download-file-base
-
 [instance-eggs]
 recipe = zc.recipe.egg
 eggs =
-  ${python-PyYAML:egg}
   toml
 
+
+[instance-agent]
+<= download-file-base
+
+[instance-default]
+<= download-file-base
+
 [instance-profile]
 recipe = slapos.recipe.template:jinja2
 url = ${:_profile_base_location_}/${:filename}
@@ -145,6 +148,8 @@ output = ${buildout:directory}/instance.cfg
 extensions = jinja2.ext.do
 context =
   section buildout buildout
+  key instance_default instance-default:target
+  key instance_agent instance-agent:target
   key openssl_bin openssl-output:openssl
   key telegraf_bin gowork:telegraf-bin
   key telegraf_input_slapos_bin gowork:telegraf-input-slapos-bin
@@ -157,13 +162,12 @@ context =
   key curl_bin :curl-bin
   key dash_bin :dash-bin
   key jq_bin :jq-bin
+  key caucase_jinja2_library caucase-jinja2-library:target
 curl-bin = ${curl:location}/bin/curl
 dash-bin = ${dash:location}/bin/dash
 jq-bin = ${jq:location}/bin/jq
 depends = ${instance-eggs:eggs} ${caucase-eggs:eggs}
-import-list =
-  file caucase caucase-jinja2-library:target
+
 
 [versions]
-inotifyx = 0.2.2
 toml = 0.10.2

software/grafana/test/test.py

@@ -53,8 +53,7 @@ class GrafanaTestCase(SlapOSInstanceTestCase):
   Since the instances takes time to start and stop,
   we increase the number of retries.
   """
-#  instance_max_retry = 50
-  instance_max_retry = 30  # TODO
+  instance_max_retry = 50
   report_max_retry = 30
 
 
@@ -117,6 +116,8 @@ class TestGrafana(GrafanaTestCase):
       if loki_health.get('data'):
         break
       time.sleep(retry)
+    else:
+      self.fail(loki_health)
     self.assertEqual(loki_health['status'], "success")
     self.assertIn("app", loki_health['data'])
 
@@ -132,19 +133,20 @@ class TestGrafana(GrafanaTestCase):
 
 class TestGrafanaEmailEnabled(GrafanaTestCase):
   __partition_reference__ = 'mail'
-  smtp_verify_ssl = "true"
+  smtp_verify_ssl = True
   smtp_skip_verify = "false"
 
   @classmethod
   def getInstanceParameterDict(cls):
-    return json.dumps({"_": {
-      "smtp-server": "smtp.example.com:25",
-      "smtp-username": "smtp_username",
-      "smtp-password": "smtp_password",
-      'smtp-verify-ssl': cls.smtp_verify_ssl,
-      "email-from-address": "grafana@example.com",
-      "email-from-name": "Grafana From Name",
-    }})
+    return {"_": json.dumps({
+      "email": {
+        "smtp-server": "smtp.example.com:25",
+        "smtp-username": "smtp_username",
+        "smtp-password": "smtp_password",
+        'smtp-verify-ssl': cls.smtp_verify_ssl,
+        "email-from-address": "grafana@example.com",
+        "email-from-name": "Grafana From Name",
+      }})}
 
   def test_email_enabled(self):
     config = configparser.ConfigParser()
@@ -163,7 +165,7 @@ class TestGrafanaEmailEnabled(GrafanaTestCase):
 
 
 class TestGrafanaEmailEnabledSkipVerify(TestGrafanaEmailEnabled):
-  smtp_verify_ssl = "false"
+  smtp_verify_ssl = False
   smtp_skip_verify = "true"
 
 
@@ -212,9 +214,14 @@ class TestTelegraf(GrafanaTestCase):
             "instance-root": cls.slap._instance_root,
             "partitions": [
               {
-                "name": "test grafana - partition name",
+                "name": "test grafana - default partition",
                 "type": "default",
-                "reference": "G0"
+                "reference": "G0",  # XXX assumes partitions will be allocated in order
+              },
+              {
+                "name": "test grafana - agent partition",
+                "type": "default",
+                "reference": "G1"
               },
             ],
           },
@@ -223,6 +230,10 @@ class TestTelegraf(GrafanaTestCase):
     }
     return {'_': json.dumps(parameter_dict)}
 
+  def setUp(self):
+    self.connection_params = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
+    self.influxdb_url = self.connection_params['influxdb-url']
+
   def test_telegraf_running(self):
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
@@ -230,9 +241,6 @@ class TestTelegraf(GrafanaTestCase):
     self.assertEqual(process_info['statename'], 'RUNNING')
 
   def test_telegraf_ingest_slapos_metrics(self):
-    self.connection_params = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
-    self.influxdb_url = self.connection_params['influxdb-url']
-
     # wait for data to be ingested
     time.sleep(16)
 
@@ -264,27 +272,27 @@ class TestTelegraf(GrafanaTestCase):
       if resp.ok and resp.json()['results'][0].get('series'):
         break
       time.sleep(i)
+    else:
+      self.fail(resp.text)
 
     series = resp.json()['results'][0].get('series')
-    print(series)
-    breakpoint()
+
     # hashes and "-on-watch" is removed from process_name
-    self.asserIn('grafana', [s['tags']['process_name'] for s in series])
-    self.asserIn('telegraf', [s['tags']['process_name'] for s in series])
-    self.asserIn('loki-service', [s['tags']['process_name'] for s in series])
-    self.asserIn('loki-grafana-client-certificate-updater', [s['tags']['process_name'] for s in series])
+    self.assertIn('grafana', [s['tags']['process_name'] for s in series])
+    self.assertIn('telegraf', [s['tags']['process_name'] for s in series])
+    self.assertIn('loki-service', [s['tags']['process_name'] for s in series])
+    self.assertIn('loki-grafana-client-certificate-updater', [s['tags']['process_name'] for s in series])
 
-    tags = [s['tags'] for s in series][0]
-    self.assertEqual(tags['name'], 'test grafana - partition name')
+    tags = [s['tags'] for s in series if s['tags']['partition_reference'] == 'G0'][0]
+    self.assertEqual(tags['name'], 'test grafana - default partition')
     self.assertEqual(tags['computer_id'], self.slap._computer_id)
+    self.assertEqual(tags['partition_reference'], 'G0')
 
     self.assertEqual(
-      set([s['tags']['partition_reference'] for s in series]),
-      {'G0'},
+      {s['tags']['partition_reference'] for s in series},
+      {'G0', 'G1'},
     )
 
-    self.fail('TODO')
-
 
 class TestLoki(GrafanaTestCase):
   @classmethod
@@ -296,12 +304,10 @@ class TestLoki(GrafanaTestCase):
         "applications": [
           {
             "name": "TestLoki",
-         #   "instance-root": "/",  # XXX needed ?
             "partitions": [
               {
-                # no slapos for system application
                 "name": "test log file",
-                "log-file-patterns": cls._logfile.name,
+                "log-file-patterns": [cls._logfile.name],
                 "static-tags": {
                   "testtag": "foo",
                 },
@@ -352,14 +358,17 @@ class TestLoki(GrafanaTestCase):
         if result := resp.json().get('data', {}).get('result', []):
           break
       time.sleep(i)
+    else:
+      self.fail(resp.text)
     self.assertEqual(
       result[0]['stream'],
       {
         'app': 'TestLoki',
+        'computer_id': self.slap._computer_id,
         'detected_level': 'info',
         'filename': self._logfile.name,
-        'job': 'test log file',
-        'partition': 'test log file',
+        'job': 'TestLoki-test log file',
+        'name': 'test log file',
         'service_name': 'TestLoki',
         'testtag': 'foo',
       }
@@ -404,13 +413,13 @@ class TestListenInPartition(GrafanaTestCase):
             c.laddr for c in self.process_dict['influxdb'].connections()
             if c.status == 'LISTEN'
         ]),
-        [
+        sorted([
             (self._ipv4_address, 8088),
             (self.computer_partition_ipv6_address, 8086),
-        ],
+        ]),
     )
 
-  def test_telegraph_listen(self):
+  def test_telegraf_listen(self):
     self.assertEqual(
         [
             c.laddr for c in self.process_dict['telegraf'].connections()
@@ -425,10 +434,10 @@ class TestListenInPartition(GrafanaTestCase):
             c.laddr for c in self.process_dict['loki-service'].connections()
             if c.status == 'LISTEN'
         ]),
-        [
-            (self.computer_partition_ipv6_address, 3100),
+        sorted([
             (self._ipv4_address, 9095),
-        ],
+            (self.computer_partition_ipv6_address, 3100),
+        ]),
     )
 
   def test_promtail_listen(self):