X Checkpoint on embedded postgresql

slapos/recipe/postgres/__init__.py

@@ -29,6 +29,7 @@ import md5
 import os
 import subprocess
 import textwrap
+import shutil
 from zc.buildout import UserError
 
 from slapos.recipe.librecipe import GenericBaseRecipe
@@ -79,11 +80,19 @@ class Recipe(GenericBaseRecipe):
         # if the pgdata already exists, skip all steps, we don't need to do anything.
 
         if not os.path.exists(pgdata):
+            try:
                 self.createCluster()
                 self.createConfig()
                 self.createDatabase()
                 self.updateSuperuser()
                 self.createRunScript()
+            except:
+                # do not leave half-installed postgresql - else next time we
+                # run we won't update it.
+                shutil.rmtree(pgdata)
+                raise
+
+
 
         # install() methods usually return the pathnames of managed files.
         # If they are missing, they will be rebuilt.

software/gitlab/instance-gitlab.cfg.in

@@ -9,7 +9,6 @@ parts =
     publish-instance-info
 
 
-#   request-postgresql
     gitlab-rails
     gitlab-rake
     gitlab-unicorn
@@ -17,8 +16,10 @@ parts =
 
     gitlab-root-shadow
 
-
     service-nginx
+    service-postgresql
+#   service-redis
+
 
 # std stuff for slapos instance
 eggs-directory = {{ eggs_directory }}
@@ -68,33 +69,35 @@ recipe  = slapos.cookbook:publish
 url     = ${gitlab-backend:url}
 
 
-# TODO reintegrate postgresql/redis into this instance
-#################################
-#   need: PostgreSQL instance   #
-#################################
-[request-postgresql]
-<= slap-connection
-recipe  = slapos.cookbook:request
-# TODO name = $(gitlab-name)/pgsql
-name    = g-pgsql
-software-url = {{ urlparse.urljoin(software_release_url, '../postgres/software.cfg') }}
-return  = url
-
-[postgresql-urlparse]
-recipe  = slapos.cookbook:urlparse
-url     = ${request-postgresql:connection-url}
-
-
-############################
-#   need: Redis instance   #
-############################
-[request-redis]
-<= slap-connection
-recipe  = slapos.cookbook:request
-# TODO name = $(gitlab-name)/pgsql
-name    = g-redis
-software-url = {{ urlparse.urljoin(software_release_url, '../redis-server/software.cfg') }}
-return  = redis_ip redis_port
+{#
+# # TODO reintegrate postgresql/redis into this instance
+# #################################
+# #   need: PostgreSQL instance   #
+# #################################
+# [request-postgresql]
+# <= slap-connection
+# recipe  = slapos.cookbook:request
+# # TODO name = $(gitlab-name)/pgsql
+# name    = g-pgsql
+# software-url = {{ urlparse.urljoin(software_release_url, '../postgres/software.cfg') }}
+# return  = url
+# 
+# [postgresql-urlparse]
+# recipe  = slapos.cookbook:urlparse
+# url     = ${request-postgresql:connection-url}
+# 
+# 
+# ############################
+# #   need: Redis instance   #
+# ############################
+# [request-redis]
+# <= slap-connection
+# recipe  = slapos.cookbook:request
+# # TODO name = $(gitlab-name)/pgsql
+# name    = g-redis
+# software-url = {{ urlparse.urljoin(software_release_url, '../redis-server/software.cfg') }}
+# return  = redis_ip redis_port
+#}
 
 
 
@@ -161,7 +164,8 @@ rendered= ${gitlab:etc}/${:_buildout_section_name_}
 <= gitlab-etc-template
 template= {{ database_yml_in }}
 context =
-    section pgsql   postgresql-urlparse
+    section gitlab_backend      gitlab-backend
+    section pgsql               service-postgresql
 
 [gitlab.yml]
 <= gitlab-etc-template
@@ -174,8 +178,8 @@ context =
 [resque.yml]
 <= gitlab-etc-template
 template= {{ resque_yml_in }}
-context =
-    section redis   request-redis
+#context =
+#    section redis   request-redis
 
 [smtp_settings.rb]
 <= gitlab-etc-template
@@ -326,6 +330,52 @@ command =
 #   sidekiq
 
 
+#####################
+#   Postgresql db   #
+#####################
+
+#[postgresql]
+#recipe  = slapos.cookbook:mkdirectory
+#srv     = ${directory:srv}/postgresql
+# XXX etc   - no
+# XXX log
+
+[postgresql-password]
+recipe  = slapos.cookbook:generate.password
+storage-path = ...
+
+
+# XXX gitlab tunes:
+# - shared_buffers
+# - work_mem
+# - checkpoint_*
+# - effective_check_size
+# - lc_* en_US.UTF-8 -> C  (?)
+[service-postgresql]
+recipe  = slapos.cookbook:postgres
+bin     = {{ postgresql_location }}/bin
+services= ${directory:service}
+
+dbname  = gitlabhq_production
+superuser = gitlab
+# TODO password
+password= ...
+
+# empty addresses - listn only on unix socket
+ipv4    = !py!set([])
+ipv6    = !py!set([])
+ipv6-random =
+port    =
+
+pgdata-directory = ${directory:srv}/postgresql
+
+
+
+#############
+#   Redis   #
+#############
+
+
 ######################
 #   Nginx frontend   #
 ######################

software/gitlab/instance.cfg.in

@@ -34,6 +34,7 @@ context =
     raw git                         ${git:location}/bin/git
     raw nginx_bin                   ${nginx-output:nginx}
     raw mime_types                  ${nginx-output:mime}
+    raw postgresql_location         ${postgresql92:location}
 
     raw macrolib_cfg_in             ${macrolib.cfg.in:target}
     raw gitlab_parameters_cfg       ${gitlab-parameters.cfg:target}

software/gitlab/software.cfg

@@ -5,6 +5,7 @@ extends =
     ../../component/ruby/buildout.cfg
     ../../component/golang/buildout.cfg
     ../../component/postgresql/buildout.cfg
+    ../../component/redis/buildout.cfg
     ../../component/cmake/buildout.cfg
     ../../component/icu/buildout.cfg
     ../../component/pkgconfig/buildout.cfg
@@ -16,6 +17,7 @@ parts =
     ruby2.1
     git
     postgresql92
+    redis28
     cmake
     icu
     pkgconfig

software/gitlab/template/database.yml.in

@@ -6,11 +6,10 @@
 production:
   adapter: postgresql
   encoding: unicode
-  #database: gitlabhq_production
-  database: {{ pgsql.path }}
+  database: {{ pgsql.dbname }}
   pool: 10
-  username: '{{ pgsql.username }}'
-  password: '{{ pgsql.password }}'
-  host:     {{ pgsql.host }}
-  port:     {{ pgsql.port }}
-  # socket:
+  username: '{{ pgsql.superuser }}'
+  password:
+  host:     '{{ pgsql["pgdata-directory"] }}'
+  port:
+  socket:

software/gitlab/template/resque.yml.in

@@ -3,4 +3,5 @@
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
 
-production: redis://[{{ redis['connection-redis_ip'] }}]:{{ redis['connection-redis_port'] }}
+{# production: redis://[{{ redis['connection-redis_ip'] }}]:{{ redis['connection-redis_port'] }} #}
+production: redis://127.0.0.1:12345