Go1.11 + fix helloweb + change helloweb-go to be done via gowork

While helping @romain to review my current work we tried to build helloworld and found that it is completely broken.

It was first minimally fixed to pin eggs and the like.

Then I've added Go1.11 to our stack, so that recent-enough compiler/stdlib could be used. A note goes here that currently our lab uses go1.9 which is outdated and not supported by upstream, and other bits are still using go1.10.3 while go1.10.7 was already released some time ago with bug and security fixes. I did not touched neither  go1.10 / nor go1.9, but imho it makes sense for infrastructure people to look into appropriate upgrades.

Finally component/helloweb/ is switched to use gowork infrastructure (see 1b540151 for gowork introduction). Today gowork is used to build eveything go-related, so building go bits manually via cmmi as example is  a bit misleading. Gowork is also required in case we'll need to use any third-party or our package a helloweb-go.

/cc @luke, @alain.takoudjou
/reviewed-by @jerome, @tomo
/reviewed-on nexedi/slapos!505

component/golang/buildout.cfg

@@ -53,6 +53,15 @@ md5sum = d15dfb264105c5e84fbe33f4a4aa5021
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
+[golang1.11]
+<= golang-common
+url = https://dl.google.com/go/go1.11.4.src.tar.gz
+md5sum = a77697673215be465d1b583680ef2318
+
+# go1.11 needs go1.4 to bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}
+
 
 # ---- infrastructure to build Go workspaces / projects ----
 

component/helloweb/buildout.cfg

@@ -4,6 +4,7 @@ extends =
   ../git/buildout.cfg
   ../ruby/buildout.cfg
   ../golang/buildout.cfg
+  gowork.cfg
 
 parts =
   helloweb-python
@@ -11,20 +12,40 @@ parts =
   helloweb-go
 
 
-# repository with examples
-[helloweb-repository]
-recipe  = slapos.recipe.build:gitclone
-git-executable = ${git:location}/bin/git
-repository = https://lab.nexedi.com/nexedi/helloweb.git
-revision = 39fd89a35718c3b4828296efcd9a45b6c9bbd13b
-location = ${buildout:parts-directory}/helloweb
+# helloweb.git repository with examples.
+#
+# We clone it via golang infrastructure into a Go workspace.
+# Via gowork helloweb/go is not only cloned, but also built and installed into gowork/bin.
+#
+# Another option would be to use slapos.recipe.build:gitclone and handle
+# cloning+building manually. However to be able to use third-party Go packages
+# we need gowork support, and other languages can use helloweb repository from
+# under gowork as well.
+[gowork]
+install =
+    lab.nexedi.com/nexedi/helloweb/go/...
+
+golang  = ${golang1.11:location}
+
+
+# -*- go -*-
+[helloweb-go]
+# we already have gowork/bin/helloweb with helloweb Go build.
+# Add bin/helloweb-go that runs go version of helloweb without any environment preset needed.
+recipe  = collective.recipe.template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+input   = inline:
+    #!/bin/sh
+    . ${gowork:env.sh}
+    exec helloweb "$@"
 
 
 # -*- python -*-
 [helloweb-egg]
 recipe  = zc.recipe.egg:develop
 egg     = helloweb
-setup   = ${helloweb-repository:location}/python/
+setup   = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/python/
 
 [helloweb-python]
 recipe  = zc.recipe.egg:scripts
@@ -47,10 +68,10 @@ scripts = helloweb=helloweb-python
 [bundler]
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 recipe  = rubygemsrecipe
-url     = https://rubygems.org/rubygems/rubygems-2.4.8.zip
+url     = https://rubygems.org/rubygems/rubygems-2.5.2.zip
 ruby-location = ${ruby2.1:location}
 ruby-executable = ${:ruby-location}/bin/ruby
-gems    = bundler==1.10.6
+gems    = bundler==1.11.2
 
 # bin installed here
 bundle  = ${buildout:bin-directory}/bundle
@@ -66,7 +87,7 @@ environment =
 
 [helloweb-ruby-bundle]
 recipe  = slapos.recipe.cmmi
-path    = ${helloweb-repository:location}/ruby/
+path    = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/ruby/
 
 configure-command = :
 make-binary =
@@ -84,15 +105,3 @@ input   = inline:
     exec ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_} "$@"
 
 
-# -*- go -*-
-[helloweb-go]
-recipe  = slapos.recipe.cmmi
-path    = ${helloweb-repository:location}/go/
-
-go      = ${golang19:location}/bin/go
-configure-command = :
-make-binary =
-make-targets= cd ${:path} &&
-     ${:go} build
-        -o ${buildout:bin-directory}/${:_buildout_section_name_}
-        helloweb.go

component/helloweb/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_lab.nexedi.com_nexedi_helloweb:recipe}
+
+
+[go_lab.nexedi.com_nexedi_helloweb]
+<= go-git-package
+go.importpath = lab.nexedi.com/nexedi/helloweb
+repository    = https://lab.nexedi.com/nexedi/helloweb.git
+revision      = 8bfedac656

component/ruby/buildout.cfg

@@ -20,8 +20,8 @@ configure-options =
   --with-out-ext=tk,tk/tkutil,win32,win32ole
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
-  CPPFLAGS=-I${gdbm:location}/include -I${libffi:location}/include -I${libyaml:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${openssl:location}/include -I${readline:location}/include -I${zlib:location}/include
-  LDFLAGS=-L${gdbm:location}/lib -L${libffi:location}/lib -L${libyaml:location}/lib -L${ncurses:location}/lib -L${openssl:location}/lib -L${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${gdbm:location}/lib  -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${libyaml:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${zlib:location}/lib
+  CPPFLAGS=-I${gdbm:location}/include -I${libffi:location}/include -I${libyaml:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${openssl-1.0:location}/include -I${readline:location}/include -I${zlib:location}/include
+  LDFLAGS=-L${gdbm:location}/lib -L${libffi:location}/lib -L${libyaml:location}/lib -L${ncurses:location}/lib -L${openssl-1.0:location}/lib -L${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${gdbm:location}/lib  -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${libyaml:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${zlib:location}/lib
   PKG_CONFIG_PATH=${libyaml:location}/lib/
 
 

software/helloworld/software.cfg

@@ -38,3 +38,8 @@ mode = 0644
 extensions = jinja2.ext.do
 context =
   section buildout  buildout
+
+# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
+[versions]
+slapos.recipe.template = 4.3
+rubygemsrecipe  = 0.2.2+slapos001

software/helloworld/test/README.md

+Tests for HelloWorld software release

software/helloworld/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.helloworld'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' helloworld",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/helloworld/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('SLAPOS_TEST_DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class HelloWorldTestCase(utils.SlapOSInstanceTestCase):
+  # to be defined by subclasses
+  name = None
+  kind = None
+
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')),)
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {"name": cls.name}
+
+
+class HTTPRequestTestMixin(object):
+  """Test that the service url.${kind} responds Hello ${name}
+  """
+  def test_get(self):
+    url = self.computer_partition.getConnectionParameterDict()['url.{}'.format(
+        self.kind)]
+    response = requests.get(url)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    self.assertTrue(
+        response.text.startswith("Hello {}".format(self.name)), response.text)
+
+
+class TestPython(HelloWorldTestCase, HTTPRequestTestMixin):
+  name = "Python"
+  kind = "python"
+
+
+class TestRuby(HelloWorldTestCase, HTTPRequestTestMixin):
+  name = "Ruby"
+  kind = "ruby"
+
+
+class TestGolang(HelloWorldTestCase, HTTPRequestTestMixin):
+  name = "Go"
+  kind = "go"

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = b98a93c66e1875bbbfc7e08ee0efc502
+md5sum = 30f0e0c5b88b518047197f251f0aea8d

software/slapos-sr-testing/instance.cfg

@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp
 
 [test-list]
-path_list = ${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup}
+path_list = ${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup}
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

software/slapos-sr-testing/software.cfg

@@ -71,6 +71,11 @@ setup = ${slapos-repository:location}/software/seleniumserver/test/
 egg = slapos.test.slaprunner
 setup = ${slapos-repository:location}/software/slaprunner/test/
 
+[slapos.test.helloworld-setup]
+<= setup-develop-egg
+egg = slapos.test.helloworld
+setup = ${slapos-repository:location}/software/helloworld/test/
+
 [erp5.util-setup]
 <= setup-develop-egg
 egg = erp5.util[testnode]