instance-pull-backup.cfg.in 11.4 KB
Newer Older
1 2 3
[buildout]

parts =
4
  publish-connection-information
5 6
  pbs
  logrotate
7
  logrotate-entry-notifier
8 9
  cron
  cron-entry-logrotate
10
  pbs-sshkeys-authority
11
  sshkeys-openssh
12
  backup-checksum-integrity-promise
13
  resilient-genstatrss-wrapper
14
  pbs-push-history-log
15
  backup-signature-link
16
  cron-pbs-status-feed
17
  pull-push-stalled-promise
18
  notifier-feed-status-promise
19

20
## Monitor for pbs
21
  monitor-base
22
  monitor-check-resilient-feed-file
23

24
extends = ${monitor2-template:rendered}
25 26 27 28
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true

29

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
#----------------
#--
#-- Creation of all needed directories.

[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
var = $${buildout:directory}/var

[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
45
services = $${rootdirectory:etc}/service
46 47 48
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
49
ssh-home = $${rootdirectory:etc}/ssh
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
notifier = $${rootdirectory:etc}/notifier

[directory]
recipe = slapos.cookbook:mkdirectory
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
cronoutput = $${basedirectory:log}/cron-ouput
pbs-backup = $${basedirectory:backup}/pbs
sshkeys = $${rootdirectory:srv}/sshkeys
pbs-wrappers = $${rootdirectory:bin}/pbs
dot-ssh = $${basedirectory:ssh-home}/.ssh
notifier-feeds = $${basedirectory:notifier}/feeds
notifier-callbacks = $${basedirectory:notifier}/callbacks
66
notifier-status-items = $${basedirectory:notifier}/status-items
67
monitor-resilient = $${monitor-directory:private}/resilient
68 69 70 71 72


#----------------
#--
#-- Set up the equeue and notifier.
73

74 75 76
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
77
lockfile = $${basedirectory:run}/equeue.lock
78 79 80 81 82
log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue

83 84 85 86 87 88
[notifier-port]
recipe = slapos.cookbook:free_port
minimum = 8088
maximum = 8097
ip = $${notifier:host}

89 90
# notifier.notify adds the [exporter, notifier] to the execution queue
# notifier.notify.callback sets up a callback
91
[notifier]
92 93 94 95 96 97
recipe = slapos.recipe.template:jinja2
template = ${template-wrapper:output}
rendered = $${:wrapper}
wrapper = $${basedirectory:services}/notifier
mode = 0700
command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
98
host = $${slap-network-information:global-ipv6}
99
port = $${notifier-port:port}
100 101 102
context =
  key content notifier:command

103 104 105 106 107 108 109
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
110

111

112 113 114 115
#----------------
#--
#-- The pull-backup-server contains every backup (incremental)
#-- to prevent a corrupt dump from destroying everything.
116

117 118 119
[pbs]
<= notifier
recipe = slapos.cookbook:pbs
120
client = true
121 122 123 124
feeds = $${directory:notifier-feeds}
callbacks = $${directory:notifier-callbacks}
equeue-socket = $${equeue:socket}
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
125
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
126
sshclient-binary = $${openssh-client:wrapper-path}
127 128
known-hosts = $${directory:dot-ssh}/known_hosts
promises-directory = $${basedirectory:promises}
129
directory = $${directory:pbs-backup}
130 131
cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers}
132
run-directory = $${basedirectory:run}
133
pull-push-maximum-run = 5
134 135
# XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
136
slave-instance-list = $${slap-parameter:slave_instance_list}
137
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
138 139
# To get a verbose feed about PBS state
instance-root-name = $${instance-info-parameters:root-name}
140
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
141
status-item-directory = $${directory:notifier-status-items}
142

143 144
[pbs-resilient-status-feed]
recipe = slapos.cookbook:wrapper
145 146
command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${instance-info-parameters:root-name}-PBS" --link $${pbs:log-url}
feed-path = $${directory:monitor-resilient}/pbs-status-rss
147 148 149 150 151 152
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py

[cron-pbs-status-feed]
<= cron
recipe = slapos.cookbook:cron.d
name = resilient-pbs-status-feed
153
frequency = */5 * * * *
154
command = $${pbs-resilient-status-feed:wrapper-path}
155

156 157 158
#----------------
#--
#-- Deploy cron.
159 160 161 162 163 164 165 166 167 168

[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond

169 170 171 172 173 174 175 176 177 178
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log


#----------------
#--
#-- Deploy logrotate.

179 180 181 182 183 184 185 186 187 188
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}

[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
189
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
190 191 192 193 194 195 196 197 198
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status

199
[logrotate-entry-cron]
200 201 202 203 204 205 206
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = cron
log = $${cron-simplelogger:log}
frequency = daily
rotate-num = 30

207 208 209 210 211 212 213 214 215 216 217 218
[logrotate-entry-notifier]
recipe = collective.recipe.template
mode = 600
input = inline:
  $${directory:notifier-feeds}/* {
    rotate 5
    weekly
    nocompress
    missingok
    olddir $${directory:logrotate-backup}
  }
output = $${logrotate:logrotate-entries}/notifier
219

220 221 222
#----------------
#--
#-- sshkeys
223

224
[sshkeys-directory]
225
recipe = slapos.cookbook:mkdirectory
226 227
requests = $${directory:sshkeys}/openssl-requests
keys = $${directory:sshkeys}/openssl-keys
228

229
[pbs-sshkeys-authority]
230 231 232
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
233
wrapper = $${basedirectory:services}/pbs_sshkeys_authority
234
keygen-binary = ${openssh:location}/bin/ssh-keygen
235

236
[sshkeys-openssh]
237
<= pbs-sshkeys-authority
238
recipe = slapos.cookbook:sshkeys_authority.request
239
name = pbs-client
240
type = rsa
241 242 243
executable = $${openssh-client:wrapper-path}
public-key = $${openssh-client:identity-file}.pub
private-key = $${openssh-client:identity-file}
244
wrapper = $${rootdirectory:bin}/do_backup
245 246


247 248
#----------------
#--
249
#-- OpenSSH.
250

251 252
[openssh-client]
recipe = slapos.cookbook:wrapper
253
home = $${basedirectory:ssh-home}
254 255 256 257
identity-file = $${:home}/id_rsa
command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file}
wrapper-path = $${rootdirectory:bin}/ssh
parameters-extra = true
258 259 260 261 262


#----------------
#--
#-- Slave instance list (empty default).
263 264 265 266 267
[htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:etc}/.monitor_user
bytes = 8
username = admin
268 269 270

[slap-parameter]
slave_instance_list = []
271
ignore-known-hosts-file = false
272 273 274 275 276
monitor-cors-domains = 
monitor-httpd-port = 8070
monitor-title = PBS Instance
monitor-password = $${htpasswd:passwd}
monitor-username = $${htpasswd:username}
277

278 279 280 281
#----------------
#--
#-- Resiliency promises.

282
[backup-checksum-integrity-promise]
283 284
recipe = slapos.recipe.template:jinja2
template = inline:
285 286 287 288 289 290 291 292 293
  #!${dash:location}/bin/dash
  # Raise an error if signatures are different
  # Error cannot be deduced if files do not exist
  cd $${directory:pbs-backup}
  if [ ! -f "proof.signature" ]; then exit 0; fi
  backup_signature=$(find . -maxdepth 2 -name backup.signature)
  if [ -z "$backup_signature" ]; then
    exit 0;
  else
294 295 296 297 298 299 300
    diff -q "proof.signature" "$backup_signature";
    if [ "$?" -eq 0 ]; then
      exit 0;
    else
      echo "Signature file is not the same before and after transfer"
      exit 1
    fi
301
  fi
302
rendered = $${basedirectory:promises}/backup-checksum-integrity
303 304
mode = 700

305 306 307 308 309
[resilient-genstatrss-wrapper]
recipe = slapos.cookbook:wrapper
# XXX - hard-coded Urls
command-line = ${buildout:directory}/bin/rdiffbackup.genstatrss --output '$${monitor-directory:public}/resilient-feed' --rdiff_backup_data_folder '$${pbs:rdiff-backup-data-folder}' --feed_url '$${monitor-conf-parameters:base-url}/public/resilient-feed'
wrapper-path = $${directory:bin}/resilient-genstatrss.py
310

311 312 313 314
[pbs-push-history-log]
recipe = cns.recipe.symlink
symlink = $${pbs:rdiff-backup-data-folder}/restore.log = $${basedirectory:log}/pbs-push-history-log

315 316
[backup-signature-link]
recipe = cns.recipe.symlink
317
symlink = $${directory:pbs-backup}/proof.signature = $${directory:monitor-resilient}/backup.signature
318

319 320
[pull-push-stalled-promise]
recipe = slapos.cookbook:wrapper
321 322
# # time-buffer is 24h (+1h of latitude)
command-line = ${buildout:bin-directory}/check-feed-as-promise --feed-path $${pbs-resilient-status-feed:feed-path} --title --ok-pattern 'OK' --time-buffer 90000
323 324
wrapper-path = $${basedirectory:promises}/stalled-pull-push

325 326 327 328 329 330 331 332 333 334
[notifier-feed-status-promise]
recipe = slapos.recipe.template:jinja2
template = ${notifier-feed-promise-template:target}
rendered = $${basedirectory:promises}/notifier-feed-check-malformed-or-failure.py
mode = 700
context =
  key notifier_feed_directory directory:notifier-feeds
  raw base_url http://[$${notifier:host}]:$${notifier:port}/get/
  raw python_executable ${buildout:executable}

335 336 337 338
#----------------
#--
#-- Publish instance parameters.

339
[publish-connection-information]
340
recipe = slapos.cookbook:publish
341
ssh-key = $${sshkeys-openssh:public-key-value}
342 343
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
344 345 346 347
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
348 349 350 351 352

#----------------
#--
#-- Monitor

353 354 355 356 357 358
[monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
monitor-title = $${slap-parameter:monitor-title}
cors-domains = $${slap-parameter:monitor-cors-domains}
username = $${slap-parameter:monitor-username}
password = $${slap-parameter:monitor-password}
359

360 361 362 363
[monitor-conf-parameters]
private-path-list += 
  $${directory:logrotate-backup}
  $${basedirectory:log}
364 365 366 367

[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
368
rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
369 370 371
mode = 700
context =
  key input_feed_directory directory:notifier-feeds
372
  key monitor_feed_directory monitor-directory:public
373
  key base_url publish-connection-information:feeds-url
374
  raw python_executable ${buildout:executable}