Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Guillaume Hervier
slapos.core
Commits
8f6917f9
Commit
8f6917f9
authored
Mar 13, 2015
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
allow to format and configure external data storage folder
parent
b1b19629
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
78 additions
and
9 deletions
+78
-9
slapos/format.py
slapos/format.py
+35
-1
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+31
-7
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+9
-1
slapos/grid/templates/buildout-tail.cfg.in
slapos/grid/templates/buildout-tail.cfg.in
+3
-0
No files found.
slapos/format.py
View file @
8f6917f9
...
@@ -218,6 +218,7 @@ class Computer(object):
...
@@ -218,6 +218,7 @@ class Computer(object):
"Object representing the computer"
"Object representing the computer"
instance_root
=
None
instance_root
=
None
software_root
=
None
software_root
=
None
instance_storage_home
=
None
def
__init__
(
self
,
reference
,
interface
=
None
,
addr
=
None
,
netmask
=
None
,
def
__init__
(
self
,
reference
,
interface
=
None
,
addr
=
None
,
netmask
=
None
,
ipv6_interface
=
None
,
software_user
=
'slapsoft'
,
ipv6_interface
=
None
,
software_user
=
'slapsoft'
,
...
@@ -386,6 +387,7 @@ class Computer(object):
...
@@ -386,6 +387,7 @@ class Computer(object):
tap
=
Tap
(
partition_dict
[
'reference'
])
tap
=
Tap
(
partition_dict
[
'reference'
])
address_list
=
partition_dict
[
'address_list'
]
address_list
=
partition_dict
[
'address_list'
]
external_storage_list
=
partition_dict
.
get
(
'external_storage_list'
,
[])
partition
=
Partition
(
partition
=
Partition
(
reference
=
partition_dict
[
'reference'
],
reference
=
partition_dict
[
'reference'
],
...
@@ -393,6 +395,7 @@ class Computer(object):
...
@@ -393,6 +395,7 @@ class Computer(object):
user
=
user
,
user
=
user
,
address_list
=
address_list
,
address_list
=
address_list
,
tap
=
tap
,
tap
=
tap
,
external_storage_list
=
external_storage_list
,
)
)
computer
.
partition_list
.
append
(
partition
)
computer
.
partition_list
.
append
(
partition
)
...
@@ -464,6 +467,15 @@ class Computer(object):
...
@@ -464,6 +467,15 @@ class Computer(object):
os
.
chown
(
slapsoft
.
path
,
slapsoft_pw
.
pw_uid
,
slapsoft_pw
.
pw_gid
)
os
.
chown
(
slapsoft
.
path
,
slapsoft_pw
.
pw_uid
,
slapsoft_pw
.
pw_gid
)
os
.
chmod
(
self
.
software_root
,
0o755
)
os
.
chmod
(
self
.
software_root
,
0o755
)
# get list of instance external storage if exist
instance_external_list
=
[]
if
self
.
instance_storage_home
:
# XXX - Hard limit for storage number to 4
for
i
in
range
(
1
,
5
):
storage_path
=
os
.
path
.
join
(
self
.
instance_storage_home
,
'data%s'
%
i
)
if
os
.
path
.
exists
(
storage_path
):
instance_external_list
.
append
(
storage_path
)
tap_address_list
=
[]
tap_address_list
=
[]
if
alter_network
and
self
.
tap_gateway_interface
and
create_tap
:
if
alter_network
and
self
.
tap_gateway_interface
and
create_tap
:
gateway_addr_dict
=
getIfaceAddressIPv4
(
self
.
tap_gateway_interface
)
gateway_addr_dict
=
getIfaceAddressIPv4
(
self
.
tap_gateway_interface
)
...
@@ -481,6 +493,8 @@ class Computer(object):
...
@@ -481,6 +493,8 @@ class Computer(object):
partition
.
path
=
os
.
path
.
join
(
self
.
instance_root
,
partition
.
reference
)
partition
.
path
=
os
.
path
.
join
(
self
.
instance_root
,
partition
.
reference
)
partition
.
user
.
setPath
(
partition
.
path
)
partition
.
user
.
setPath
(
partition
.
path
)
partition
.
user
.
additional_group_list
=
[
slapsoft
.
name
]
partition
.
user
.
additional_group_list
=
[
slapsoft
.
name
]
partition
.
external_storage_list
=
[
'%s/%s'
%
(
path
,
partition
.
reference
)
for
path
in
instance_external_list
]
if
alter_user
:
if
alter_user
:
partition
.
user
.
create
()
partition
.
user
.
create
()
...
@@ -510,6 +524,7 @@ class Computer(object):
...
@@ -510,6 +524,7 @@ class Computer(object):
# Reconstructing partition's directory
# Reconstructing partition's directory
partition
.
createPath
(
alter_user
)
partition
.
createPath
(
alter_user
)
partition
.
createExternalPath
(
alter_user
)
# Reconstructing partition's address
# Reconstructing partition's address
# There should be two addresses on each Computer Partition:
# There should be two addresses on each Computer Partition:
...
@@ -555,7 +570,7 @@ class Computer(object):
...
@@ -555,7 +570,7 @@ class Computer(object):
class
Partition
(
object
):
class
Partition
(
object
):
"Represent a computer partition"
"Represent a computer partition"
def
__init__
(
self
,
reference
,
path
,
user
,
address_list
,
tap
):
def
__init__
(
self
,
reference
,
path
,
user
,
address_list
,
tap
,
external_storage_list
=
[]
):
"""
"""
Attributes:
Attributes:
reference: String, the name of the partition.
reference: String, the name of the partition.
...
@@ -563,6 +578,7 @@ class Partition(object):
...
@@ -563,6 +578,7 @@ class Partition(object):
user: User, the user linked to this partition.
user: User, the user linked to this partition.
address_list: List of associated IP addresses.
address_list: List of associated IP addresses.
tap: Tap, the tap interface linked to this partition.
tap: Tap, the tap interface linked to this partition.
external_storage_list: Base path list of folder to format for data storage
"""
"""
self
.
reference
=
str
(
reference
)
self
.
reference
=
str
(
reference
)
...
@@ -570,6 +586,7 @@ class Partition(object):
...
@@ -570,6 +586,7 @@ class Partition(object):
self
.
user
=
user
self
.
user
=
user
self
.
address_list
=
address_list
or
[]
self
.
address_list
=
address_list
or
[]
self
.
tap
=
tap
self
.
tap
=
tap
self
.
external_storage_list
=
[]
def
__getinitargs__
(
self
):
def
__getinitargs__
(
self
):
return
(
self
.
reference
,
self
.
path
,
self
.
user
,
self
.
address_list
,
self
.
tap
)
return
(
self
.
reference
,
self
.
path
,
self
.
user
,
self
.
address_list
,
self
.
tap
)
...
@@ -589,6 +606,21 @@ class Partition(object):
...
@@ -589,6 +606,21 @@ class Partition(object):
os
.
chown
(
self
.
path
,
owner_pw
.
pw_uid
,
owner_pw
.
pw_gid
)
os
.
chown
(
self
.
path
,
owner_pw
.
pw_uid
,
owner_pw
.
pw_gid
)
os
.
chmod
(
self
.
path
,
0o750
)
os
.
chmod
(
self
.
path
,
0o750
)
def
createExternalPath
(
self
,
alter_user
=
True
):
"""
Create and external directory of the partition, assign to the partition user
and give it the 750 permission. In case if path exists just modifies it.
"""
for
path
in
self
.
external_storage_list
:
storage_path
=
os
.
path
.
abspath
(
path
)
owner
=
self
.
user
if
self
.
user
else
User
(
'root'
)
if
not
os
.
path
.
exists
(
storage_path
):
os
.
mkdir
(
storage_path
,
0o750
)
if
alter_user
:
owner_pw
=
pwd
.
getpwnam
(
owner
.
name
)
os
.
chown
(
storage_path
,
owner_pw
.
pw_uid
,
owner_pw
.
pw_gid
)
os
.
chmod
(
storage_path
,
0o750
)
class
User
(
object
):
class
User
(
object
):
"""User: represent and manipulate a user on the system."""
"""User: represent and manipulate a user on the system."""
...
@@ -1167,6 +1199,7 @@ def do_format(conf):
...
@@ -1167,6 +1199,7 @@ def do_format(conf):
computer
.
instance_root
=
conf
.
instance_root
computer
.
instance_root
=
conf
.
instance_root
computer
.
software_root
=
conf
.
software_root
computer
.
software_root
=
conf
.
software_root
computer
.
instance_storage_home
=
conf
.
instance_storage_home
conf
.
logger
.
info
(
'Updating computer'
)
conf
.
logger
.
info
(
'Updating computer'
)
address
=
computer
.
getAddress
(
conf
.
create_tap
)
address
=
computer
.
getAddress
(
conf
.
create_tap
)
computer
.
address
=
address
[
'addr'
]
computer
.
address
=
address
[
'addr'
]
...
@@ -1208,6 +1241,7 @@ class FormatConfig(object):
...
@@ -1208,6 +1241,7 @@ class FormatConfig(object):
software_user
=
None
software_user
=
None
tap_gateway_interface
=
None
tap_gateway_interface
=
None
use_unique_local_address_block
=
None
use_unique_local_address_block
=
None
instance_storage_home
=
None
def
__init__
(
self
,
logger
):
def
__init__
(
self
,
logger
):
self
.
logger
=
logger
self
.
logger
=
logger
...
...
slapos/grid/SlapObject.py
View file @
8f6917f9
...
@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch'
...
@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch'
REQUIRED_COMPUTER_PARTITION_PERMISSION
=
0o750
REQUIRED_COMPUTER_PARTITION_PERMISSION
=
0o750
CP_STORAGE_FOLDER_NAME
=
'DATA'
# XXX not very clean. this is changed when testing
# XXX not very clean. this is changed when testing
PROGRAM_PARTITION_TEMPLATE
=
pkg_resources
.
resource_stream
(
__name__
,
PROGRAM_PARTITION_TEMPLATE
=
pkg_resources
.
resource_stream
(
__name__
,
'templates/program_partition_supervisord.conf.in'
).
read
()
'templates/program_partition_supervisord.conf.in'
).
read
()
...
@@ -341,7 +343,8 @@ class Partition(object):
...
@@ -341,7 +343,8 @@ class Partition(object):
logger
,
logger
,
certificate_repository_path
=
None
,
certificate_repository_path
=
None
,
retention_delay
=
'0'
,
retention_delay
=
'0'
,
instance_min_free_space
=
None
instance_min_free_space
=
None
,
instance_storage_home
=
''
):
):
"""Initialisation of class parameters"""
"""Initialisation of class parameters"""
self
.
buildout
=
buildout
self
.
buildout
=
buildout
...
@@ -358,6 +361,7 @@ class Partition(object):
...
@@ -358,6 +361,7 @@ class Partition(object):
self
.
partition_id
=
partition_id
self
.
partition_id
=
partition_id
self
.
server_url
=
server_url
self
.
server_url
=
server_url
self
.
software_release_url
=
software_release_url
self
.
software_release_url
=
software_release_url
self
.
instance_storage_home
=
instance_storage_home
self
.
key_file
=
''
self
.
key_file
=
''
self
.
cert_file
=
''
self
.
cert_file
=
''
...
@@ -512,6 +516,7 @@ class Partition(object):
...
@@ -512,6 +516,7 @@ class Partition(object):
'software_release_url'
:
self
.
software_release_url
,
'software_release_url'
:
self
.
software_release_url
,
'key_file'
:
self
.
key_file
,
'key_file'
:
self
.
key_file
,
'cert_file'
:
self
.
cert_file
,
'cert_file'
:
self
.
cert_file
,
'storage_home'
:
self
.
instance_storage_home
,
}
}
open
(
config_location
,
'w'
).
write
(
buildout_text
)
open
(
config_location
,
'w'
).
write
(
buildout_text
)
os
.
chmod
(
config_location
,
0o640
)
os
.
chmod
(
config_location
,
0o640
)
...
@@ -684,12 +689,22 @@ class Partition(object):
...
@@ -684,12 +689,22 @@ class Partition(object):
sr_symlink
=
os
.
path
.
join
(
self
.
instance_path
,
'software_release'
)
sr_symlink
=
os
.
path
.
join
(
self
.
instance_path
,
'software_release'
)
if
os
.
path
.
islink
(
sr_symlink
):
if
os
.
path
.
islink
(
sr_symlink
):
os
.
unlink
(
sr_symlink
)
os
.
unlink
(
sr_symlink
)
data_base_link
=
os
.
path
.
join
(
self
.
instance_path
,
CP_STORAGE_FOLDER_NAME
)
for
root
,
dirs
,
file_list
in
os
.
walk
(
self
.
instance_path
):
if
self
.
instance_storage_home
and
os
.
path
.
exists
(
data_base_link
)
and
\
for
directory
in
dirs
:
os
.
path
.
isdir
(
data_base_link
):
shutil
.
rmtree
(
os
.
path
.
join
(
self
.
instance_path
,
directory
))
for
filename
in
os
.
listdir
(
data_base_link
):
for
file
in
file_list
:
data_symlink
=
os
.
path
.
join
(
data_base_link
,
filename
)
os
.
remove
(
os
.
path
.
join
(
self
.
instance_path
,
file
))
partition_data_path
=
os
.
path
.
join
(
self
.
instance_storage_home
,
filename
,
self
.
partition_id
)
if
os
.
path
.
lexists
(
data_symlink
):
os
.
unlink
(
data_symlink
)
if
os
.
path
.
exists
(
partition_data_path
):
self
.
cleanupFolder
(
partition_data_path
)
self
.
cleanupFolder
(
self
.
instance_path
)
# Cleanup all Data storage location of this partition
if
os
.
path
.
exists
(
self
.
supervisord_partition_configuration_path
):
if
os
.
path
.
exists
(
self
.
supervisord_partition_configuration_path
):
os
.
remove
(
self
.
supervisord_partition_configuration_path
)
os
.
remove
(
self
.
supervisord_partition_configuration_path
)
...
@@ -699,6 +714,15 @@ class Partition(object):
...
@@ -699,6 +714,15 @@ class Partition(object):
return
True
return
True
def
cleanupFolder
(
self
,
folder_path
):
"""Delete all files and folders in a specified directory
"""
for
root
,
dirs
,
file_list
in
os
.
walk
(
folder_path
):
for
directory
in
dirs
:
shutil
.
rmtree
(
os
.
path
.
join
(
folder_path
,
directory
))
for
file
in
file_list
:
os
.
remove
(
os
.
path
.
join
(
folder_path
,
file
))
def
fetchInformations
(
self
):
def
fetchInformations
(
self
):
"""Fetch usage informations with buildout, returns it.
"""Fetch usage informations with buildout, returns it.
"""
"""
...
...
slapos/grid/slapgrid.py
View file @
8f6917f9
...
@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger):
...
@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger):
# Try to fetch from deprecated argument
# Try to fetch from deprecated argument
computer_partition_filter_list
=
op
.
get
(
'only-cp'
,
op
.
get
(
'only_cp'
)),
computer_partition_filter_list
=
op
.
get
(
'only-cp'
,
op
.
get
(
'only_cp'
)),
software_min_free_space
=
software_min_free_space
,
software_min_free_space
=
software_min_free_space
,
instance_min_free_space
=
instance_min_free_space
)
instance_min_free_space
=
instance_min_free_space
,
instance_storage_home
=
op
.
get
(
'instance_storage_home'
))
def
check_required_only_partitions
(
existing
,
required
):
def
check_required_only_partitions
(
existing
,
required
):
...
@@ -286,6 +287,7 @@ class Slapgrid(object):
...
@@ -286,6 +287,7 @@ class Slapgrid(object):
computer_partition_filter_list
=
None
,
computer_partition_filter_list
=
None
,
software_min_free_space
=
None
,
software_min_free_space
=
None
,
instance_min_free_space
=
None
,
instance_min_free_space
=
None
,
instance_storage_home
=
None
,
):
):
"""Makes easy initialisation of class parameters"""
"""Makes easy initialisation of class parameters"""
# Parses arguments
# Parses arguments
...
@@ -337,6 +339,10 @@ class Slapgrid(object):
...
@@ -337,6 +339,10 @@ class Slapgrid(object):
self
.
maximum_periodicity
=
maximum_periodicity
self
.
maximum_periodicity
=
maximum_periodicity
self
.
software_min_free_space
=
software_min_free_space
self
.
software_min_free_space
=
software_min_free_space
self
.
instance_min_free_space
=
instance_min_free_space
self
.
instance_min_free_space
=
instance_min_free_space
if
instance_storage_home
:
self
.
instance_storage_home
=
os
.
path
.
abspath
(
instance_storage_home
)
else
:
self
.
instance_storage_home
=
""
def
_getWatchdogLine
(
self
):
def
_getWatchdogLine
(
self
):
invocation_list
=
[
WATCHDOG_PATH
]
invocation_list
=
[
WATCHDOG_PATH
]
...
@@ -644,6 +650,7 @@ class Slapgrid(object):
...
@@ -644,6 +650,7 @@ class Slapgrid(object):
logger
=
self
.
logger
,
logger
=
self
.
logger
,
retention_delay
=
retention_delay
,
retention_delay
=
retention_delay
,
instance_min_free_space
=
self
.
instance_min_free_space
,
instance_min_free_space
=
self
.
instance_min_free_space
,
instance_storage_home
=
self
.
instance_storage_home
,
)
)
computer_partition_state
=
computer_partition
.
getState
()
computer_partition_state
=
computer_partition
.
getState
()
...
@@ -1102,6 +1109,7 @@ class Slapgrid(object):
...
@@ -1102,6 +1109,7 @@ class Slapgrid(object):
certificate_repository_path
=
self
.
certificate_repository_path
,
certificate_repository_path
=
self
.
certificate_repository_path
,
buildout
=
self
.
buildout
,
buildout
=
self
.
buildout
,
logger
=
self
.
logger
,
logger
=
self
.
logger
,
instance_storage_home
=
self
.
instance_storage_home
,
)
)
local_partition
.
stop
()
local_partition
.
stop
()
try
:
try
:
...
...
slapos/grid/templates/buildout-tail.cfg.in
View file @
8f6917f9
...
@@ -22,4 +22,7 @@ server_url = %(server_url)s
...
@@ -22,4 +22,7 @@ server_url = %(server_url)s
software_release_url = %(software_release_url)s
software_release_url = %(software_release_url)s
key_file = %(key_file)s
key_file = %(key_file)s
cert_file = %(cert_file)s
cert_file = %(cert_file)s
[storage-configuration]
storage-home = %(storage_home)s
# This is end of zc.builodout profile's tail added by slapgrid
# This is end of zc.builodout profile's tail added by slapgrid
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment