Skip to content

slapos
slapos
Commit 11ac4e0e authored by Hardik Juneja's avatar Hardik Juneja Committed by Rafael Monnerat
1 Browse files
Options

stack/erp5: Add promises to check mariadb and apachedex reports

parent b6481107
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 7 deletions
stack/erp5/buildout.cfg
View file @ 11ac4e0e
......@@ -52,6 +52,7 @@ extends =
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/dash/buildout.cfg
../../component/bash/buildout.cfg
../../component/wget/buildout.cfg
../../component/aspell/buildout.cfg
../../component/cloudooo/buildout.cfg
......@@ -100,6 +101,7 @@ parts +=
percona-toolkit
zabbix-agent
dash
bash
wget
userhosts
postfix
......@@ -168,6 +170,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
<= download-base
mode = 755
[mariadb-slowquery-check-script]
<= download-base
mode = 755
[mariadb-slow-query-report-script]
<= download-base
mode = 755
......@@ -232,6 +238,8 @@ context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key apache_location apache:location
key apdex_result_check_script apdex-result-check-script:target
key mariadb_slowquery_check_script mariadb-slowquery-check-script:target
key aspell_location aspell:location
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
......@@ -242,6 +250,7 @@ context =
key curl_location curl:location
key cyrus_sasl_location cyrus-sasl:location
key dash_location dash:location
key bash_location bash:location
key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location
......@@ -351,6 +360,9 @@ link-binary =
[template-balancer]
<= download-base
[apdex-result-check-script]
<= download-base
[template-haproxy-cfg]
<= download-base
......
stack/erp5/buildout.hash.cfg
View file @ 11ac4e0e
......@@ -19,11 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = dc974bd74cf967ae6250d81322629c44
md5sum = 48fb6a0ed34cfc6de6b40acd7088c51c
[mariadb-slowquery-check-script]
filename = instance-mariadb-check-slowquery-result.sh.in
md5sun = c9cd394f3d031cef77da50610544d2b0
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 2126e64bc70893500dc24c29770e84c8
md5sum = 9c34cba998ce0ed3b96e2604d2ed1cb5
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -75,7 +79,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template]
filename = instance.cfg.in
md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
md5sum = cbe1df5dbd9b79ed7adc027fd183e186
[monitor-template-dummy]
filename = dummy.cfg
......@@ -95,7 +99,11 @@ md5sum = a2377d5c53fd2a441ea713b428e4844b
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 25d0ebecda78d1440dacd3495d53c928
md5sum = 79fc39f7fbf13b1788adb5c33150dd80
[apdex-result-check-script]
filename = instance-balancer-check-apachedex-result.sh.in
md5sum = e1dd2f2884ea0e9f6f230419e032d927
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer-check-apachedex-result.sh.in 0 → 100644
View file @ 11ac4e0e
#!{{ bash }}
set -e
apachedex_file='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
apachedex_status_file={{ apdex_status_file }}
desired_threshold={{ default_threshold }}
user_threshold=$(<{{ user_threshold }})
if [ ! -z "$user_threshold" ]
then
desired_threshold=$user_threshold
fi
# Check if the file is there
if [ ! -s "$apachedex_file" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$apachedex_file" ]; then
touch $apachedex_file
else
modified_date=`stat -c '%Z' $apachedex_file`
current_date=`date +%s`
if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
json_content=`cat $apachedex_status_file`
message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $message
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
regex="apdex threshold<\/th><td>(.*)s<\/td>"
file_content=`cat $apachedex_file`
if [[ $file_content =~ $regex ]]
then
threshold=${BASH_REMATCH[1]}
threshold=${threshold:-0}
if [[ `echo "$threshold > $desired_threshold" | bc` -eq 1 ]]
then
echo "Your score is $threshold, Thanks for keeping it all clean"
else
echo "Threshold is lower than exptected: Expected was $desired_threshold and we current is $threshold"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the apdex result"
}
fi
stack/erp5/instance-balancer.cfg.in
View file @ 11ac4e0e
......@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command = apachedex_every_3_hour
[apachedex-parameters]
......@@ -299,6 +299,19 @@ default = ${monitor-directory:etc}/apdex_default
base-list = ${monitor-directory:etc}/apdex_base_list
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
apdex-promise-threshold = ${monitor-directory:etc}/apdex-promise-threshold
[{{ section('monitor-promise-apachedex-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['apdex-result-check-script'] }}
rendered = ${monitor-directory:promises}/check-apachedex-result
status-file = ${monitor-directory:private}/apachedex.report.json
context =
raw default_threshold 0.7
raw bash {{ parameter_dict['bash'] }}/bin/bash
  • Vincent Pelletier @vpelletier ·
    Owner

    Although this is less dangerous, this use of raw is again wrong: concatenate & dumps in jinja in apachedex-parameters section and reference with key.

Please register or sign in to reply
key apdex_file directory:apachedex
key apdex_status_file :status-file
key user_threshold apachedex-parameters:apdex-promise-threshold
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
......@@ -310,6 +323,7 @@ instance-configuration =
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
file apachedex-erp5-promise-threshold ${apachedex-parameters:apdex-promise-threshold}
[buildout]
extends =
......
stack/erp5/instance-mariadb-check-slowquery-result.sh.in 0 → 100644
View file @ 11ac4e0e
#!{{ bash }}
set -e
digest_file='{{ slow_query_digest }}/slowquery_digest.txt'
slow_query_status_file='{{ slow_query_status }}'
desired_max_query_threshold={{ default_threshold }}
desired_slow_query_threshold=0
max_queries_threshold='{{ max_queries_threshold }}'
if [ ! -z "$max_queries_threshold" ]
then
desired_max_query_threshold=$max_queries_threshold
fi
slowest_queries_threshold='{{ slowest_queries_threshold }}'
if [ ! -z "$slowest_queries_threshold" ]
then
desired_slow_query_threshold=$slowest_queries_threshold
fi
# Check if the file is there
if [ ! -s "$digest_file" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$digest_file" ]; then
touch $digest_file
else
modified_date=`stat -c '%Z' $digest_file`
current_date=`date +%s`
if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
json_content=`cat $slow_query_status_file`
message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $message
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
# get the total number of queries run and the max time
# TODO: improve regex
# TODO: improve the parameters (currently we are using threshold on queries and max execute time)
# # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
regex="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
file_content=`cat $digest_file`
if [[ $file_content =~ $regex ]]
then
max_query_threshold=${BASH_REMATCH[1]}
slowest_query_threshold=${BASH_REMATCH[2]}
hasK="${max_query_threshold: -1}"
if [[ "$hasK" == "k" ]]
then
pre="${max_query_threshold::-1}"
max_query_threshold=$(echo "scale=4; ${pre:-0}*1000" | bc)
else
max_query_threshold=${max_query_threshold:-0}
fi
# TODO: support ms
slowest_query_threshold="${slowest_query_threshold:-0}"
if [[ `echo "$max_query_threshold < $desired_max_query_threshold" | bc` -eq 1 && `echo "$slowest_query_threshold > $desired_slow_query_threshold" | bc` -eq 1 ]]
then
echo "Total number of slow queries are: $max_query_threshold"
echo "Time taken by slowest query is: $slowest_query_threshold"
echo "Thanks for keeping it all clean"
else
echo "Ops! One of the two expected parameters did not meet"
echo "Time taken by slowest query is: $slowest_query_threshold s and required maximum is $desired_slow_query_threshold s"
echo "Total slow queries are $max_query_threshold and expected maximum value is $desired_max_query_threshold"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the result"
}
fi
stack/erp5/instance-mariadb.cfg.in
View file @ 11ac4e0e
......@@ -283,7 +283,7 @@ recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
filename = mariadb_slow_query_every_2_minutes
mode = 755
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
......@@ -291,6 +291,23 @@ context =
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder :output-folder
[slowquery-parameters]
max-queries-threshold = ${monitor-directory:etc}/max-slowqueries-threshold
slowest-queries-threshold = ${monitor-directory:etc}/slowest-queries-threshold
[{{ section('monitor-promise-slowquery-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
context =
raw default_threshold 0.7
raw bash {{ parameter_dict['bash'] }}/bin/bash
raw slow_query_digest ${directory:srv}/monitor
key slow_query_status :status-file
key max_queries_threshold slowquery-parameters:max-queries-thershold
key slowest_queries_threshold slowquery-parameters:slowest-queries-threshold
[{{ section('promise') }}]
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
......@@ -302,6 +319,9 @@ monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file max-queries-threshold ${slowquery-parameteres:max-queries-threshold}
file slowest-queries-threshold ${slowquery-parameters:slowest-queries-threshold}
[buildout]
extends =
......
stack/erp5/instance.cfg.in
View file @ 11ac4e0e
......@@ -93,6 +93,7 @@ openssl-location = {{ openssl_location }}
[dynamic-template-balancer-parameters]
apache = {{ apache_location }}
apdex-result-check-script = {{ apdex_result_check_script }}
openssl = {{ openssl_location }}
haproxy = {{ haproxy_location }}
bin-directory = {{ bin_directory }}
......@@ -101,6 +102,7 @@ run-apachedex-location = {{ bin_directory }}/runApacheDex
6tunnel = {{ sixtunnel_location }}
curl-location = {{ curl_location }}
dash = {{ dash_location }}
bash = {{ bash_location }}
template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }}
template-monitor = {{ dumps(template_monitor) }}
......@@ -169,6 +171,7 @@ extra-context =
section parameter_dict dynamic-template-kumofs-parameters
[dynamic-template-mariadb-parameters]
bash = {{ bash_location }}
coreutils-location = {{ coreutils_location }}
dash-location = {{ dash_location }}
findutils-location = {{ findutils_location }}
......@@ -180,6 +183,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
percona-tools-location = {{ percona_toolkit_location }}
template-monitor = {{ template_monitor }}
......
stack/erp5/mysql-querydigest.sh.in
View file @ 11ac4e0e
......@@ -10,6 +10,8 @@ if [ ! -d "$OUTPUT_FOLDER" ]; then
exit 0
fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
......@@ -19,4 +21,4 @@ if [ ! -f "$SLOW_LOG" ]; then
exit 1
fi
eval $PT_QUERY_EXEC $SLOW_LOG
eval $PT_QUERY_EXEC $SLOW_LOG | tee $OUTPUT_FILE
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7