Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Boris Kocherov
slapos
Commits
95ac5087
Commit
95ac5087
authored
8 years ago
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
monitor stack: reduce amount of history files
parent
ca60ce6f
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
58 additions
and
38 deletions
+58
-38
stack/monitor/scripts/globalstate.py
stack/monitor/scripts/globalstate.py
+1
-0
stack/monitor/scripts/monitor.py
stack/monitor/scripts/monitor.py
+16
-17
stack/monitor/scripts/run-promise.py
stack/monitor/scripts/run-promise.py
+41
-21
No files found.
stack/monitor/scripts/globalstate.py
View file @
95ac5087
...
...
@@ -104,6 +104,7 @@ def main(args_list):
'success'
:
success
,
'warning'
:
warning
,
},
type
=
'global'
,
date
=
latest_date
,
_links
=
{
"rss_url"
:
{
"href"
:
"%s/public/feed"
%
base_url
},
"public_url"
:
{
"href"
:
"%s/share/jio_public/"
%
base_url
},
...
...
This diff is collapsed.
Click to expand it.
stack/monitor/scripts/monitor.py
View file @
95ac5087
...
...
@@ -97,7 +97,6 @@ class Monitoring(object):
self
.
report_script_folder
=
config
.
get
(
"monitor"
,
"report-folder"
)
self
.
webdav_url
=
'%s/share'
%
config
.
get
(
"monitor"
,
"base-url"
)
self
.
public_url
=
'%s/public'
%
config
.
get
(
"monitor"
,
"base-url"
)
self
.
status_history_folder
=
os
.
path
.
join
(
self
.
public_folder
,
'history'
)
self
.
python
=
config
.
get
(
"monitor"
,
"python"
)
or
"python"
self
.
public_path_list
=
config
.
get
(
"monitor"
,
"public-path-list"
).
split
()
self
.
private_path_list
=
config
.
get
(
"monitor"
,
"private-path-list"
).
split
()
...
...
@@ -107,7 +106,7 @@ class Monitoring(object):
self
.
parameter_cfg_file
=
config
.
get
(
"monitor"
,
"parameter-file-path"
).
strip
()
self
.
config_folder
=
os
.
path
.
join
(
self
.
private_folder
,
'config'
)
self
.
report_folder
=
os
.
path
.
join
(
self
.
private_folder
,
'report'
)
self
.
report_folder
=
self
.
private_folder
self
.
promise_dict
=
{}
for
promise_folder
in
self
.
promise_folder_list
:
...
...
@@ -237,14 +236,14 @@ class Monitoring(object):
else
:
response
=
urllib2
.
urlopen
(
url
)
except
urllib2
.
HTTPError
:
return
'Unknow Instance'
return
'Unknow
n
Instance'
else
:
try
:
monitor_dict
=
json
.
loads
(
response
.
read
())
return
monitor_dict
.
get
(
'title'
,
'Unknow Instance'
)
return
monitor_dict
.
get
(
'title'
,
'Unknow
n
Instance'
)
except
ValueError
,
e
:
print
"Bad Json file at %s"
%
url
return
'Unknow Instance'
return
'Unknow
n
Instance'
def
configureFolders
(
self
):
# configure public and private folder
...
...
@@ -256,7 +255,6 @@ class Monitoring(object):
jio_private
=
os
.
path
.
join
(
self
.
webdav_folder
,
'jio_private'
)
mkdirAll
(
jio_public
)
mkdirAll
(
jio_private
)
mkdirAll
(
self
.
status_history_folder
)
try
:
os
.
symlink
(
self
.
public_folder
,
os
.
path
.
join
(
jio_public
,
'.jio_documents'
))
except
OSError
,
e
:
...
...
@@ -269,11 +267,9 @@ class Monitoring(object):
raise
self
.
data_folder
=
os
.
path
.
join
(
self
.
private_folder
,
'data'
,
'.jio_documents'
)
self
.
report_folder
=
os
.
path
.
join
(
self
.
report_folder
,
'.jio_documents'
)
config_folder
=
os
.
path
.
join
(
self
.
config_folder
,
'.jio_documents'
)
mkdirAll
(
self
.
data_folder
)
mkdirAll
(
config_folder
)
mkdirAll
(
self
.
report_folder
)
try
:
os
.
symlink
(
os
.
path
.
join
(
self
.
private_folder
,
'data'
),
os
.
path
.
join
(
jio_private
,
'data'
))
...
...
@@ -285,11 +281,6 @@ class Monitoring(object):
except
OSError
,
e
:
if
e
.
errno
!=
os
.
errno
.
EEXIST
:
raise
try
:
os
.
symlink
(
self
.
report_folder
,
os
.
path
.
join
(
jio_private
,
'report'
))
except
OSError
,
e
:
if
e
.
errno
!=
os
.
errno
.
EEXIST
:
raise
def
makeConfigurationFiles
(
self
):
config_folder
=
os
.
path
.
join
(
self
.
config_folder
,
'.jio_documents'
)
...
...
@@ -397,7 +388,7 @@ class Monitoring(object):
def
generateReportCronEntries
(
self
):
cron_line_list
=
[]
# We should add the possibility to modify this parameter later from monitor interface
report_frequency
=
"*/
3
0 * * * *"
report_frequency
=
"*/
2
0 * * * *"
report_name_list
=
[
name
.
replace
(
'.report.json'
,
''
)
for
name
in
os
.
listdir
(
self
.
report_folder
)
if
name
.
endswith
(
'.report.json'
)]
...
...
@@ -451,7 +442,7 @@ class Monitoring(object):
for
service_name
,
promise
in
self
.
promise_items
:
service_config
=
promise
[
"configuration"
]
service_status_path
=
"%s/%s.status.json"
%
(
self
.
public_folder
,
service_name
)
# hardcoded
service_status_path
=
"%s/%s.status.json"
%
(
self
.
public_folder
,
service_name
)
mkdirAll
(
os
.
path
.
dirname
(
service_status_path
))
promise_cmd_line
=
[
...
...
@@ -463,7 +454,7 @@ class Monitoring(object):
'--promise_script "%s"'
%
promise
[
"path"
],
'--promise_name "%s"'
%
service_name
,
'--monitor_url "%s/jio_private/"'
%
self
.
webdav_url
,
# XXX hardcoded,
'--history_folder "%s"'
%
self
.
status_history
_folder
,
'--history_folder "%s"'
%
self
.
public
_folder
,
'--instance_name "%s"'
%
self
.
title
,
'--hosting_name "%s"'
%
self
.
root_title
]
...
...
@@ -539,13 +530,21 @@ class Monitoring(object):
# Rotate monitor data files
option_list
=
[
'daily'
,
'nocreate'
,
'noolddir'
,
'rotate
30
'
,
'daily'
,
'nocreate'
,
'noolddir'
,
'rotate
5
'
,
'nocompress'
,
'extension .json'
,
'dateext'
,
'dateformat -%Y-%m-%d'
,
'notifempty'
]
file_list
=
[
"%s/*.data.json"
%
self
.
data_folder
]
self
.
generateLogrotateEntry
(
'monitor.data'
,
file_list
,
option_list
)
# Rotate public history status file, delete data of previous days
option_list
=
[
'daily'
,
'nocreate'
,
'rotate 0'
,
'nocompress'
,
'notifempty'
]
file_list
=
[
"%s/*.history.json"
%
self
.
public_folder
]
self
.
generateLogrotateEntry
(
'monitor.service.status'
,
file_list
,
option_list
)
# Add cron entry for SlapOS Collect
command
=
"%s %s --output_folder %s --collector_db %s"
%
(
self
.
python
,
self
.
collect_script
,
self
.
data_folder
,
self
.
collector_db
)
...
...
This diff is collapsed.
Click to expand it.
stack/monitor/scripts/run-promise.py
View file @
95ac5087
...
...
@@ -65,6 +65,7 @@ def main():
status_json
[
'title'
]
=
parser
.
promise_name
status_json
[
'instance'
]
=
parser
.
instance_name
status_json
[
'hosting_subscription'
]
=
parser
.
hosting_name
status_json
[
'type'
]
=
parser
.
promise_type
# Save the lastest status change date (needed for rss)
status_json
[
'change-time'
]
=
ps_process
.
create_time
()
...
...
@@ -85,9 +86,7 @@ def main():
os
.
remove
(
parser
.
pid_path
)
def
updateStatusHistoryFolder
(
name
,
status_file
,
history_folder
,
promise_type
):
old_history_list
=
[]
keep_item_amount
=
25
history_path
=
os
.
path
.
join
(
history_folder
,
name
,
'.jio_documents'
)
history_path
=
os
.
path
.
join
(
history_folder
)
if
not
os
.
path
.
exists
(
status_file
):
return
if
not
os
.
path
.
exists
(
history_folder
):
...
...
@@ -101,14 +100,35 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
else
:
raise
with
open
(
status_file
,
'r'
)
as
sf
:
status_dict
=
json
.
loads
(
sf
.
read
())
filename
=
'%s.%s.json'
%
(
status_dict
[
'start-date'
].
replace
(
' '
,
'_'
).
replace
(
':'
,
''
),
promise_type
)
if
promise_type
==
'status'
:
filename
=
'%s.history.json'
%
name
history_file
=
os
.
path
.
join
(
history_path
,
filename
)
# Remove links from history (not needed)
status_dict
.
pop
(
'_links'
,
None
)
if
not
os
.
path
.
exists
(
history_file
):
with
open
(
history_file
,
'w'
)
as
f_history
:
data_dict
=
{
"date"
:
time
.
time
(),
"data"
:
[
status_dict
]
}
f_history
.
write
(
json
.
dumps
(
data_dict
))
else
:
with
open
(
history_file
,
mode
=
"r+"
)
as
f_history
:
f_history
.
seek
(
0
,
2
)
position
=
f_history
.
tell
()
-
2
f_history
.
seek
(
position
)
#f_history.write(',%s]}' % str(status_dict))
f_history
.
write
(
'%s}'
%
',{}]'
.
format
(
json
.
dumps
(
status_dict
)))
elif
promise_type
==
'report'
:
# keep_item_amount = 3
filename
=
'%s.history.json'
%
(
name
)
copyfile
(
status_file
,
os
.
path
.
join
(
history_path
,
filename
))
# Don't let history foler grow too much, keep xx files
"""
# Don't let history foler grow too much, keep xx files
file_list = filter(os.path.isfile,
glob
.
glob
(
"%s/*.%s
.json"
%
(
history_path
,
promise_type
))
glob.glob("%s/*.%s.history
.json" % (history_path, promise_type))
)
file_count = len(file_list)
if file_count > keep_item_amount:
...
...
@@ -119,7 +139,7 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
os.unlink(to_delete)
file_count -= 1
except OSError:
raise
raise"""
def
generateStatusJsonFromProcess
(
process
,
start_date
=
None
,
title
=
None
):
stdout
,
stderr
=
process
.
communicate
()
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment