repman: waitdatabases api is not stable, try to bootstrap morre often

waitdatabases can fail event if databases are ready, reduce amount of check databases
and try to bootstrap.

software/repman/buildout.hash.cfg

@@ -58,7 +58,7 @@ md5sum = c203f40a58386310a433b58fd345a341
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
-md5sum = 852dfab6d798aa1382eec4de2fd624f9
+md5sum = 70ddec7450ae8be728ec107b805fa9a6
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in

software/repman/templates/repman-manager.sh.in

 #!{{ bash_bin }}
 
-#set -e
+set -e
 
 curl () {
   {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
@@ -12,7 +12,7 @@ get_token () {
 
 wait_database () {
   NAME=$1
-  for retry in {1..50}; do
+  for retry in {1..5}; do
     echo ">> Wait until $NAME databases are ready...";
     CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
     if [ $CODE -eq 504 ]; then
@@ -29,8 +29,25 @@ wait_database () {
       fi
       echo ">> [$retry] waitdatabases returned code $CODE...";
     fi
-    sleep 30
+    sleep 15
+    echo "Reloading cluster settings..."
+    curl -H "Authorization: Bearer ${TOKEN}" \
+      {{ secure_url }}/api/clusters/$NAME/settings/actions/reload
   done
+  echo $CODE
+}
+
+check_cluster () {
+  # Check if cluster is boostrapped
+  NAME=$1
+  TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+  ERRORS=$(curl -H "Authorization: Bearer ${TOKEN}" {{ secure_url }}/api/clusters/$NAME/topology/alerts | {{ jq_bin }} -r '.errors')
+  if [ "$ERRORS" != "null" ] && [ ! -z "$ERRORS" ]; then
+    echo "ERROR: Bootstrap replication of cluster $NAME failed!";
+    echo $ERRORS;
+    return 1;
+  fi
+  return 0
 }
 
 activate_proxy () {
@@ -66,7 +83,6 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
   curl -H "Authorization: Bearer ${TOKEN}" \
     {{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
   CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
-  SUCCESS=0
   if [ $CODE -eq 200 ]; then
     activate_proxy {{ name }}
     if [ $? -eq 0 ]; then
@@ -77,6 +93,9 @@ if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
   else
     echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
   fi
+else
+  # Check cluster health
+  check_cluster {{ name }}
 fi
 
 {% endfor %}