Skip to content

erp5
erp5
Commit 8d08790c authored by Ayush Tiwari's avatar Ayush Tiwari
Browse files
Options

bt5_config: Save BM as zexp rather than a folder

parent 0d067945
Hide whitespace changes
Inline Side-by-side
Showing with 32 additions and 385 deletions
product/ERP5/Document/BusinessManager.py
View file @ 8d08790c
......@@ -246,76 +246,39 @@ class BusinessManager(XMLObject):
"""
if not self.getStatus() == 'built':
raise ValueError, 'Manager not built properly'
return self._export(path, local, bma, **kw)
f = StringIO()
self._p_jar.exportFile(self._p_oid, f)
def _export(self, path=None, local=0, bma=None, **kw):
# XXX: Improve naming
name = self.getTitle()
name = posixpath.join(path, name)
# XXX required due to overuse of os.path
name = name.replace('\\', '/').replace(':', '/')
name = quote(name + '.zexp')
obj_path = name.replace('/', os.sep)
if bma is None:
if local:
# we export into a folder tree
bma = BusinessManagerFolder(path, creation=1)
else:
# We export bm into a tarball file
if path is None:
path = self.getTitle()
bma = BusinessManagerTarball(path, creation=1)
# export bt
for prop in self.propertyMap():
prop_type = prop['type']
id = prop['id']
if id in ('id', 'uid', 'rid', 'sid', 'id_group', 'last_id', 'revision',
'install_object_list_list', 'id_generator', 'bm_for_diff'):
continue
value = self.getProperty(id)
if not value:
continue
if prop_type in ('text', 'string', 'int', 'boolean'):
bma.addObject(str(value), name=id, path='bm', ext='')
elif prop_type in ('lines', 'tokens'):
bma.addObject('\n'.join(value), name=id, path='bm', ext='')
# Export each part
for item in self._path_item_list:
item.export(context=self, bma=bma, **kw)
f.seek(0)
obj = f.read()
return bma.finishCreation()
object_path = os.path.join(path, obj_path)
path = os.path.dirname(object_path)
os.path.exists(path) or os.makedirs(path)
f = open(object_path, 'wb')
try:
f.write(obj)
finally:
f.close()
security.declareProtected(Permissions.ManagePortal, 'importFile')
def importFile(self, path):
"""
Import all xml files in Business Manager
Import Business Manager object and all attribute to current BM itself
"""
bma = (BusinessManagerFolder if os.path.isdir(path) else
BusinessManagerTarball)(path, importing=1)
bm_item = bm()
bma.importFiles(bm_item, parent=self)
prop_dict = {}
for prop in self.propertyMap():
pid = prop['id']
if pid != 'id':
prop_type = prop['type']
value = bm_item.get(pid)
if prop_type in ('text', 'string'):
prop_dict[pid] = value or ''
elif prop_type in ('int', 'boolean'):
prop_dict[pid] = value or 0
elif prop_type in ('lines', 'tokens'):
# XXX: Do add pid[:-5] after we switch to proper getters and setters
prop_dict[pid] = (value or '').splitlines()
# XXX: This is not working, needs to be fixed so as it copies all the
# properties from BMA to the newly created Business Manager
self._edit(**prop_dict)
self.storeTemplateData()
#workflow_tool = self.getPortalObject().portal_workflow
#workflow_tool.business_package_building_workflow.notifyWorkflowMethod(
# self, 'edit', kw={'comment': 'Downloaded'})
for item_object in self._path_item_list:
item_object.importFile(bma, parent=self)
# Set the status to uninstalled
self.setStatus('uninstalled')
connection = self.aq_parent._p_jar
file = open(path, 'rb')
obj = connection.importFile(file)
self._path_item_list = obj._path_item_list
self._setTemplatePathList(obj.getTemplatePathList())
def __add__(self, other):
"""
......@@ -1078,190 +1041,6 @@ class BusinessItem(Implicit, Persistent):
obj.deletePdfContent()
return obj
def export(self, context, bma, **kw):
"""
Export the business item object : fill the BusinessManagerArchive with
objects exported as XML, hierarchicaly organised.
"""
if not self._value:
return
path = self.__class__.__name__ + '/'
# We now will add the XML object and its sha hash while exporting the object
# to Business Manager itself
# Back compatibility with filesystem Documents
key = self._path
obj = self._value
if isinstance(obj, str):
if not key.startswith(path):
key = path + key
bma.addObject(obj, name=key, ext='.py')
else:
try:
extension, unicode_data, record_id = \
SEPARATELY_EXPORTED_PROPERTY_DICT[obj.__class__.__name__]
except KeyError:
pass
else:
while 1: # not a loop
obj = obj._getCopy(context)
data = getattr(aq_base(obj), record_id, None)
if unicode_data:
if type(data) is not unicode:
break
try:
data = data.encode(aq_base(obj).output_encoding)
except (AttributeError, UnicodeEncodeError):
break
elif type(data) is not bytes:
if not isinstance(data, Pdata):
break
data = bytes(data)
try:
# Delete this attribute from the object.
# in case the related Portal Type does not exist, the object may be broken.
# So we cannot delattr, but we can delete the key of its its broken state
if isinstance(obj, ERP5BaseBroken):
del obj.__Broken_state__[record_id]
obj._p_changed = 1
else:
delattr(obj, record_id)
except (AttributeError, KeyError):
# property was acquired on a class,
# do nothing, only .xml metadata will be exported
break
# export a separate file with the data
if not extension:
extension = self.guessExtensionOfDocument(obj, key, data
if record_id == 'data'
else None)
bma.addObject(StringIO(data), key, path=path,
ext='._xml' if extension == 'xml' else '.' + extension)
break
# since we get the obj from context we should
# again remove useless properties
obj = self.removeProperties(obj, 1, keep_workflow_history=True)
transaction.savepoint(optimistic=True)
f = StringIO()
obj._p_jar.exportFile(obj._p_oid, f)
bma.addObject(f, key, path=path)
def importFile(self, bma, parent, **kw):
bma.importFiles(self, parent)
def _importFile(self, file_name, file_obj, parent):
obj_key, file_ext = os.path.splitext(file_name)
# id() for installing several bt5 in the same transaction
transactional_variable_obj_key = "%s-%s" % (id(self), obj_key)
if file_ext == '.zexp':
connection = self.getConnection(parent)
__traceback_info__ = 'Importing %s' % file_name
obj = connection.importFile(file_obj, customImporters=customImporters)
self._value = obj
data = getTransactionalVariable().get(transactional_variable_obj_key)
if data is not None:
self._restoreSeparatelyExportedProperty(obj, data)
elif file_ext != '.xml':
# For ZODB Components: if .xml have been processed before, set the
# source code property, otherwise store it in a transactional variable
# so that it can be set once the .xml has been processed
data = file_obj.read()
item = parent.getBusinessItemByPath(obj_key)
obj = item._value
if obj is None:
getTransactionalVariable()[transactional_variable_obj_key] = data
else:
self._restoreSeparatelyExportedProperty(obj, data)
def _restoreSeparatelyExportedProperty(self, obj, data):
unicode_data, property_name = SEPARATELY_EXPORTED_PROPERTY_DICT[
obj.__class__.__name__][1:]
if unicode_data:
data = data.decode(obj.output_encoding)
try:
setattr(obj, property_name, data)
except BrokenModified:
obj.__Broken_state__[property_name] = data
obj._p_changed = 1
else:
# Revert any work done by __setstate__.
# XXX: This is enough for all objects we currently split in 2 files,
# but __setstate__ could behave badly with the missing attribute
# and newly added types may require more than this.
self.removeProperties(obj, 1, keep_workflow_history=True)
def getConnection(self, obj):
while True:
connection = obj._p_jar
if connection is not None:
return connection
obj = obj.aq_parent
def _compileXML(self, file):
# This method converts XML to ZEXP. Because the conversion
# is quite heavy, a persistent cache database is used to
# store ZEXP, so the second run wouldn't have to re-generate
# identical data again.
#
# For now, a pair of the path to an XML file and its modification time
# are used as a unique key. In theory, a checksum of the content could
# be used instead, and it could be more reliable, as modification time
# might not be updated in some insane filesystems correctly. However,
# in practice, checksums consume a lot of CPU time, so when the cache
# does not hit, the increased overhead is significant. In addition, it
# does rarely happen that two XML files in Business Manager contain
# the same data, so it may not be expected to have more cache hits
# with this approach.
#
# The disadvantage is that this wouldn't work with the archive format,
# because each entry in an archive does not have a mtime in itself.
# However, the plan is to have an archive to retain ZEXP directly
# instead of XML, so the idea of caching would be completely useless
# with the archive format.
name = file.name
mtime = os.path.getmtime(file.name)
key = '%s:%s' % (name, mtime)
try:
return StringIO(cache_database.db[key])
except:
pass
from Shared.DC.xml import ppml
from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata
import xml.parsers.expat
outfile=StringIO()
try:
data=file.read()
F=ppml.xmlPickler()
F.end_handlers['record'] = save_record
F.end_handlers['ZopeData'] = save_zopedata
F.start_handlers['ZopeData'] = start_zopedata
F.binary=1
F.file=outfile
p=xml.parsers.expat.ParserCreate('utf-8')
p.returns_unicode = False
p.CharacterDataHandler=F.handle_data
p.StartElementHandler=F.unknown_starttag
p.EndElementHandler=F.unknown_endtag
p.Parse(data)
try:
cache_database.db[key] = outfile.getvalue()
except:
pass
outfile.seek(0)
return outfile
except:
outfile.close()
raise
def getBusinessPath(self):
return self._path
......@@ -1283,144 +1062,6 @@ class BusinessItem(Implicit, Persistent):
def getParentBusinessManager(self):
return self.aq_parent
class BusinessManagerArchive(object):
"""
This is the base class for all Business Manager archives
"""
def __init__(self, path, **kw):
self.path = path
def addObject(self, obj, name, path=None, ext='.zexp'):
if path:
name = posixpath.join(path, name)
# XXX required due to overuse of os.path
name = name.replace('\\', '/').replace(':', '/')
name = quote(name + ext)
path = name.replace('/', os.sep)
try:
write = self._writeFile
except AttributeError:
if not isinstance(obj, str):
obj.seek(0)
obj = obj.read()
self._writeString(obj, path)
else:
if isinstance(obj, str):
obj = StringIO(obj)
else:
obj.seek(0)
write(obj, path)
def finishCreation(self):
pass
class BusinessManagerFolder(BusinessManagerArchive):
"""
Class archiving business manager into a folder tree
"""
def _writeString(self, obj, path):
object_path = os.path.join(self.path, path)
path = os.path.dirname(object_path)
os.path.exists(path) or os.makedirs(path)
f = open(object_path, 'wb')
try:
f.write(obj)
finally:
f.close()
def importFiles(self, item, parent):
"""
Import file from a local folder
"""
join = os.path.join
item_name = item.__class__.__name__
root = join(os.path.normpath(self.path), item_name, '')
root_path_len = len(root)
if CACHE_DATABASE_PATH:
try:
cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'cf')
except gdbm.error:
cache_database.db = gdbm.open(CACHE_DATABASE_PATH, 'nf')
try:
for root, dirs, files in os.walk(root):
for file_name in files:
# If the item name is Business Item, then only import the files which
# mattter for this Business Item
if item_name == 'BusinessItem':
if not fnmatch.fnmatch(file_name, '%s.*' % item._path.split('/')[-1]):
continue
file_name = join(root, file_name)
with open(file_name, 'rb') as f:
file_name = posixpath.normpath(file_name[root_path_len:])
if '%' in file_name:
file_name = unquote(file_name)
elif item_name == 'bm' and file_name == 'revision':
continue
# self.revision.hash(item_name + '/' + file_name, f.read())
f.seek(0)
item._importFile(file_name, f, parent)
finally:
if hasattr(cache_database, 'db'):
cache_database.db.close()
del cache_database.db
class BusinessManagerTarball(BusinessManagerArchive):
"""
Class archiving business manager into a tarball file
"""
def __init__(self, path, creation=0, importing=0, **kw):
super(BusinessManagerTarball, self).__init__(path, **kw)
if creation:
self.fobj = StringIO()
self.tar = tarfile.open('', 'w:gz', self.fobj)
self.time = time.time()
elif importing:
self.tar = tarfile.open(path, 'r:gz')
self.item_dict = item_dict = defaultdict(list)
for info in self.tar.getmembers():
if info.isreg():
path = info.name.split('/')
if path[0] == '.':
del path[0]
item_dict[path[1]].append(('/'.join(path[2:]), info))
def _writeFile(self, obj, path):
if self.path:
path = posixpath.join(self.path, path)
info = tarfile.TarInfo(path)
info.mtime = self.time
obj.seek(0, 2)
info.size = obj.tell()
obj.seek(0)
self.tar.addfile(info, obj)
def finishCreation(self):
self.tar.close()
return self.fobj
def importFiles(self, item, parent):
"""
Import all file from the archive to the site
"""
extractfile = self.tar.extractfile
item_name = item.__class__.__name__
for file_name, info in self.item_dict.get(item_name, ()):
if '%' in file_name:
file_name = unquote(file_name)
elif item_name == 'bm' and file_name == 'revision':
continue
f = extractfile(info)
self.revision.hash(item_name + '/' + file_name, f.read())
f.seek(0)
item._importFile(file_name, f, parent)
class bm(dict):
"""
Fake 'bm' item to read bm/* files through BusinessManagerArchive
......
product/ERP5/Tool/TemplateTool.py
View file @ 8d08790c
......@@ -387,6 +387,9 @@ class TemplateTool (BaseTool):
id = self.generateNewId()
urltype, name = splittype(url)
# Create a zexp path which would be used for Business Manager files
zexp_path = name + '/' + name.split('/')[-1] + '.zexp'
if WIN and urltype and '\\' in name:
urltype = None
name = url
......@@ -398,10 +401,13 @@ class TemplateTool (BaseTool):
del bt.uid
return self[self._setObject(id, bt)]
bt = self._download_url(url, id)
elif os.path.exists(zexp_path):
# If the path exists, we create a Business Manager object after
# downloading it from zexp path
bt = self._download_local(os.path.normpath(zexp_path), id, format_version=3)
else:
template_version_path_list = [
name+'/bp/template_format_version',
name+'/bm/template_format_version',
]
for path in template_version_path_list:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7