software/grafana: version up

update softwares to:
  - grafana 7.4.3
  - telegrah 1.17.3
  - influxdb 1.8.4
  - loki current master ( 2.1.0 )

and components to:
  - golang 1.16
  - nodejs 14.16.0
  - yarn 1.22.10
  - python 3.7

This new loki version supports parsing multi-line logs, as demonstrated in
the updated test.

Since this version we also remove cache at the end of build, which
makes the software directory smaller and accessorily prevent test failures
during the ldd checks from checkSoftware with some binaries in the cache.

software/grafana/README.md

@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions.
 See https://github.com/influxdata/telegraf to learn about plugins.
 
 Useful plugins in this context are probably
-[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec),
-[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser)
+[exec](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/exec),
+[logparser](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/logparser)
 or
-[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http).
+[http](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/http).
 
 Telegraf will save in the `telegraf` database from the embedded influxdb server.
 

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 13f4f3806522b265dd1912ff169d146d
+md5sum = 12d9507c6dfb1ae9151a0bf7eb741060
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in

software/grafana/gowork.cfg

@@ -20,22 +20,22 @@ revision      = 1f7c19e5f5
 <= go-git-package
 go.importpath = github.com/grafana/grafana
 repository    = https://github.com/grafana/grafana
-revision      = v6.3.0-0-g830da0fda0
+revision      = v7.4.3-0-g010f20c1c8
 
 [go_github.com_grafana_loki]
 <= go-git-package
 go.importpath = github.com/grafana/loki
 repository    = https://github.com/grafana/loki
-revision      = v0.3.0-0-gc673380bb3
+revision      = v2.1.0-188-g877f524c36
 
 [go_github.com_influxdata_influxdb]
 <= go-git-package
 go.importpath = github.com/influxdata/influxdb
 repository    = https://github.com/influxdata/influxdb
-revision      = v1.7.8rc1-0-ga7afa2ecf1
+revision      = v1.8.4-0-gbc8ec4384e
 
 [go_github.com_influxdata_telegraf]
 <= go-git-package
 go.importpath = github.com/influxdata/telegraf
 repository    = https://github.com/influxdata/telegraf
-revision      = 1.11.1-0-gf91de60d37
+revision      = v1.17.3-0-g24a552b90b

software/grafana/instance.cfg.in

@@ -132,7 +132,7 @@ command-line =
   bash -c "{{ influx_bin }} \
      -username ${influxdb:auth-username} \
      -password ${influxdb:auth-password} \
-     -host ${influxdb:host} \
+     -host [${influxdb:host}] \
      -port ${influxdb:http-port} \
      -unsafeSsl \
      -ssl \

software/grafana/software.cfg

@@ -22,12 +22,14 @@ parts =
   loki-config-file
   promtail-config-file
 
+[python]
+part = python3
 
 [nodejs]
-<= nodejs-10.6.0
+<= nodejs-14.16.0
 
 [yarn]
-<= yarn-1.16.0
+<= yarn-1.22.10
 
 [gowork]
 # All the softwares installed in the go workspace have "non standard" installation
@@ -44,7 +46,7 @@ loki-bin = ${:bin}/loki
 promtail-bin = ${:bin}/promtail
 
 # use recent go
-golang  = ${golang1.12:location}
+golang  = ${golang1.16:location}
 
 [gowork.goinstall]
 command = :
@@ -60,8 +62,7 @@ depends =
 command = bash -c ". ${gowork:env.sh} && \
    go install -v github.com/golang/dep/cmd/dep && \
    cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
-   dep ensure && \
-   go install ./cmd/..."
+   go install ./..."
 update-command =
 
 [telegraf-install]
@@ -75,14 +76,17 @@ update-command =
 
 [grafana-install]
 <= gowork.goinstall
-# yarn and go run build.go needs our nodejs in $PATH
-command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
-   . ${gowork:env.sh} && \
+command = bash -c ". ${gowork:env.sh} && \
    cd ${gowork:directory}/src/github.com/grafana/grafana && \
    go run build.go setup && \
    go run build.go build && \
    ${yarn:location}/bin/yarn install --pure-lockfile && \
-   ${nodejs:location}/bin/npm run build"
+   ${yarn:location}/bin/yarn run build && \
+   ${yarn:location}/bin/yarn run plugins:build-bundled && \
+   # Cleanup yarn and Cypress caches
+   rm -rf ${buildout:directory}/.cache/Cypress/ && \
+   rm -rf ${buildout:directory}/.cache/yarn/
+   "
 update-command =
 
 [loki-install]

software/grafana/test/test.py

@@ -44,11 +44,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
 class GrafanaTestCase(SlapOSInstanceTestCase):
   """Base test case for grafana.
 
-  Since the instances takes timte to start and stop,
-  we increate as lot the number of retries.
+  Since the instances takes time to start and stop,
+  we increase the number of retries.
   """
+  instance_max_retry = 50
   report_max_retry = 30
-  instance_max_retry = 30
 
 
 class TestGrafana(GrafanaTestCase):
@@ -144,21 +144,27 @@ class TestLoki(GrafanaTestCase):
                 r'''
                 - job_name: {cls.__name__}
                   pipeline_stages:
-                    - regex:
-                        expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)"
-                    - timestamp:
-                        format: 2006-01-02T15:04:05Z00:00
-                        source: timestamp
-                    - labels:
-                        level:
-                        name:
+                    - match:
+                        selector: '{{job="{cls.__name__}"}}'
+                        stages:
+                          - multiline:
+                              firstline: '^\d{{4}}-\d{{2}}-\d{{2}}\s\d{{1,2}}\:\d{{2}}\:\d{{2}}\,\d{{3}}'
+                              max_wait_time: 3s
+                          - regex:
+                              expression: '^(?P<timestamp>.*) - (?P<name>\S+) - (?P<level>\S+) - (?P<message>.*)'
+                          - timestamp:
+                              format: 2006-01-02T15:04:05Z00:00
+                              source: timestamp
+                          - labels:
+                              level:
+                              name:
                   static_configs:
-                  - targets:
-                      - localhost
-                    labels:
-                      job: {cls.__name__}
-                      __path__: {cls._logfile.name}
-                ''').format(**locals())
+                    - targets:
+                        - localhost
+                      labels:
+                        job: {cls.__name__}
+                        __path__: {cls._logfile.name}
+            ''').format(**locals())
     }
 
   @classmethod
@@ -180,6 +186,7 @@ class TestLoki(GrafanaTestCase):
     # create a logger logging to the file that we have
     # configured in instance parameter.
     test_logger = logging.getLogger(self.id())
+    test_logger.propagate = False
     test_logger.setLevel(logging.INFO)
     test_handler = logging.FileHandler(filename=self._logfile.name)
     test_handler.setFormatter(
@@ -189,16 +196,25 @@ class TestLoki(GrafanaTestCase):
     test_logger.info("testing message")
     test_logger.info("testing another message")
     test_logger.warning("testing warn")
+    # log an exception, which will be multi line in log file.
+    def nested1():
+      def nested2():
+        raise ValueError('boom')
+      nested2()
+    try:
+      nested1()
+    except ValueError:
+      test_logger.exception("testing exception")
 
     # Check our messages have been ingested
     # we retry a few times, because there's a short delay until messages are
     # ingested and returned.
-    for i in range(10):
+    for i in range(60):
       resp = requests.get(
           '{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
               **locals()),
           verify=False).json()
-      if not resp:
+      if len(resp.get('streams', [])) < 3:
         time.sleep(0.5 * i)
         continue
 
@@ -220,6 +236,15 @@ class TestLoki(GrafanaTestCase):
             line for line in info_stream['entries']
             if "testing another message" in line['line']
         ])
+
+    error_stream_list = [stream for stream in resp['streams'] if 'level="ERROR"' in stream['labels']]
+    self.assertEqual(1, len(error_stream_list), resp['streams'])
+    error_stream, = error_stream_list
+    line, = [line['line'] for line in error_stream['entries']]
+    # this entry is multi-line
+    self.assertIn('testing exception\nTraceback (most recent call last):\n', line)
+    self.assertIn('ValueError: boom', line)
+
     # The labels we have configued are also available
     resp = requests.get(
         '{self.loki_url}/api/prom/label'.format(**locals()),