Skip to content

erp5
erp5
Commit 8383758b authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Merge remote-tracking branch 'origin/master' into lazy_simulation_causality

parents 3cca5327 5161a9e3
Show whitespace changes
Inline Side-by-side
Showing with 205 additions and 137 deletions
bt5/erp5_dms/SkinTemplateItem/portal_skins/erp5_dms/ERP5Type_getSecurityCategoryMapping.xml deleted 100644 → 0
View file @ 3cca5327
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Core security script - defines the way to get security groups of the current user.\n
\n
WARNING: providing such script in erp5_dms could be dangerous\n
if this conflicts with an existing production site which uses\n
deprecated ERP5Type_asSecurityGroupIdList\n
"""\n
\n
return (\n
(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'function\'] ),\n
(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'follow_up\'] ),\n
(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'function\', \'follow_up\'] ),\n
(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'group\'] ),\n
(\'ERP5Type_getSecurityCategoryRoot\', [\'group\']),\n
)\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Type_getSecurityCategoryMapping</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_dms/bt/revision
View file @ 8383758b
1261
\ No newline at end of file
1262
\ No newline at end of file
bt5/erp5_web/SkinTemplateItem/portal_skins/erp5_web/WebPage_getStandardSubstitutionMappingDict.xml
View file @ 8383758b
......@@ -59,9 +59,16 @@
# your own script instead.\n
\n
website = context.getWebSiteValue()\n
websection = context.getWebSectionValue()\n
\n
return dict(website_url=website.absolute_url(),\n
if website is None:\n
# handle the case when substitution happens on web page module context (i.e. reindex, or mass pre conversion)\n
# then fall back to ERP5 site as a Web Site\n
website = context.getPortalObject()\n
return dict(website_url=website.absolute_url(),\n
websection_url=website.absolute_url(),\n
webpage_url=context.absolute_url())\n
else:\n
websection = context.getWebSectionValue()\n
return dict(website_url=website.absolute_url(),\n
websection_url=websection.absolute_url(),\n
webpage_url=websection.getPermanentURL(context))\n
</string> </value>
......
bt5/erp5_web/bt/revision
View file @ 8383758b
1074
\ No newline at end of file
1075
\ No newline at end of file
erp5/util/README.webchecker.txt
View file @ 8383758b
......@@ -21,6 +21,17 @@ Vary = Accept-Language, Cookie, Accept-Encoding
Accept-Language,Cookie
Expires = True
[header url=.*/sitemap]
Last-Modified = True
[header content-type=.*/javascript]
Last-Modified = True
Cache-Control = max-age=3600
Expires = True
[no_header content-type=(image/.*|text/css)]
Vary = None
[erp5_extension_list]
prohibited_file_name_list = WebSection_viewAsWeb
Base_viewHistory
......
erp5/util/webchecker/__init__.py
View file @ 8383758b
......@@ -75,7 +75,8 @@ class HTTPCacheCheckerTestSuite(object):
def __init__(self, root_url, working_directory, varnishlog_binary_path,
wget_binary_path, header_list, email_address, smtp_host,
debug_level, file_log_path):
debug_level, file_log_path, conditional_header_dict,
no_header_dict):
"""
root_url : website to check
working_directory : where fetched data will be downloaded
......@@ -92,12 +93,19 @@ class HTTPCacheCheckerTestSuite(object):
info=>normal,
warning=> nothing)
file_log_path: path to log file
conditional_header_dict : Key == Section id (like 'header url='.*/login')
value: the configuration lines in this sction
(config format is same as header_list)
no_header_dict : Key == Section id (like 'no_header url=.*/sitemap')
value: = not exsiting headers
"""
self.root_url = root_url
self.working_directory = working_directory
self.varnishlog_binary_path = varnishlog_binary_path
self.wget_binary_path = wget_binary_path
self.header_list = header_list
self.conditional_header_dict = conditional_header_dict
self.no_header_dict = no_header_dict
self.email_address = email_address
self.smtp_host = smtp_host
level = self.LOG_LEVEL_DICT.get(debug_level, logging.INFO)
......@@ -242,6 +250,119 @@ class HTTPCacheCheckerTestSuite(object):
stdout, stderr = wget_process.communicate()
return stdout
def _getHeaderPolicyList(self, url, fetched_data):
"""
create the header checking policy list by the url, and the header
[header_list]
Last-Modified = True
Vary = Accept-Language, Cookie, Accept-Encoding
Cache-Control = max-age=300
max-age=3600
[header url=.*/contact_form]
Last-Modified = True
[no-header content-type=(image/.*|.*/javascript)]
Vary = False
[erp5_extension_list]
prohibited_folder_name_list = web_page_module
document_module
prohibited_file_name_list = WebSection_viewAsWeb
Base_viewHistory
list
"""
def getNoCheckHeaderList(url, fetched_data):
"""
create no check header list
"""
pick_content_type = re.compile('^no_header\s*content-type=(.*)')
pick_url = re.compile('^no_header\s*url=(.*)')
section_list = self.no_header_dict.keys()
no_check_header_list = []
for section in section_list:
content_type_regex_str_match = pick_content_type.match(section)
url_regex_str_match = pick_url.match(section)
if content_type_regex_str_match is not None:
content_type_regex_str = content_type_regex_str_match.group(1)
content_type_regex = \
re.compile('Content-Type:\s%s' % content_type_regex_str,
re.MULTILINE | re.IGNORECASE)
if content_type_regex.search(fetched_data) is not None:
for header, value in self.no_header_dict[section]:
no_check_header_list.append(header)
continue
if url_regex_str_match is not None:
url_regex_str = url_regex_str_match.group(1)
if re.compile(url_regex_str).match(url) is not None:
for header, value in self.no_header_dict[section]:
no_check_header_list.append(header)
return no_check_header_list
def getConditionalHeaderDict(url, fetched_data):
""" create header policy by the url and header"""
conditional_header_dict = {}
section_list = self.conditional_header_dict.keys()
pick_content_type = re.compile('header\s*content-type=(.*)')
pick_url = re.compile('header\s*url=(.*)')
for section in section_list:
content_type_regex_str_match = pick_content_type.match(section)
url_regex_str_match = pick_url.match(section)
if content_type_regex_str_match is not None:
content_type_regex_str = content_type_regex_str_match.group(1)
content_type_regex = \
re.compile('Content-Type:\s%s' % content_type_regex_str,
re.MULTILINE | re.IGNORECASE)
if content_type_regex.search(fetched_data) is not None:
for header, value in self.conditional_header_dict[section]:
conditional_header_dict.setdefault(header, []).append(value)
continue
if url_regex_str_match is not None:
url_regex_str = url_regex_str_match.group(1)
if re.compile(url_regex_str).match(url) is not None:
for header, value in self.conditional_header_dict[section]:
conditional_header_dict.setdefault(header, []).append(value)
return conditional_header_dict
validator_dict = {}
no_header_list = getNoCheckHeaderList(url, fetched_data)
conditional_header_dict = getConditionalHeaderDict(url, fetched_data)
conditional_header_list = conditional_header_dict.keys()
global_header_list = self.header_list.keys()
header_policy_list = []
if conditional_header_list:
conditional_check_header_set = (set(conditional_header_list)
- set(no_header_list))
for header in conditional_check_header_set:
header_policy_list.append((header, conditional_header_dict[header]))
else:
global_check_header_set = (set(global_header_list)
- set(no_header_list))
for header in global_check_header_set:
header_policy_list.append((header, self.header_list[header]))
# return items
return header_policy_list
def _validateHeader(self, url, header, reference_value, fetched_data):
"""validate header with the header policy"""
re_compiled = re.compile(self.generic_header_search_regex % header,
re.MULTILINE | re.IGNORECASE)
match_object = re_compiled.search(fetched_data)
if match_object is None:
message = 'header:%r not found' % (header)
self.report_dict.setdefault(url, []).append(message)
else:
read_value = match_object.group(1)
if reference_value is True and not read_value:
message = 'value of header:%r not found' % (header)
self.report_dict.setdefault(url, []).append(message)
elif isinstance(reference_value, (tuple,list)):
if read_value not in reference_value:
message = 'value of header:%r does not match'\
' (%r not in %r)' %\
(header, read_value, reference_value)
self.report_dict.setdefault(url, []).append(message)
def _parseWgetLogs(self, wget_log_file, discarded_url_list=_MARKER,
prohibited_file_name_list=None,
prohibited_folder_name_list=None):
......@@ -332,24 +453,11 @@ class HTTPCacheCheckerTestSuite(object):
discarded_url_list.append(url)
else:
x_varnish_reference_list.append((x_varnish_reference, True, url))
for header, reference_value in self.header_list.iteritems():
re_compiled = re.compile(self.generic_header_search_regex % header,
re.MULTILINE | re.IGNORECASE)
match_object = re_compiled.search(fetched_data)
if match_object is None:
message = 'header:%r not found' % (header)
self.report_dict.setdefault(url, []).append(message)
else:
read_value = match_object.group(1)
if reference_value is True and not read_value:
message = 'value of header:%r not found' % (header)
self.report_dict.setdefault(url, []).append(message)
elif isinstance(reference_value, (tuple,list)):
if read_value not in reference_value:
message = 'value of header:%r does not match'\
' (%r not in %r)' %\
(header, read_value, reference_value)
self.report_dict.setdefault(url, []).append(message)
# parse the web checker configuration file and run the header
# validation method
for header, reference_value in self._getHeaderPolicyList(
url, fetched_data):
self._validateHeader(url, header, reference_value, fetched_data)
return x_varnish_reference_list, discarded_url_list[:]
def start(self, prohibited_file_name_list=None,
......@@ -428,16 +536,19 @@ def web_checker_utility():
parser.error('incorrect number of arguments')
config_path = args[0]
config = ConfigParser.RawConfigParser(defaults=dict(wget_binary_path='wget',
file_log_path='web_checker.log'))
config = ConfigParser.RawConfigParser()
config.read(config_path)
working_directory = config.get('web_checker', 'working_directory')
url = config.get('web_checker', 'url')
varnishlog_binary_path = config.get('web_checker', 'varnishlog_binary_path')
wget_binary_path = 'wget'
if config.has_option('web_checker', 'wget_binary_path'):
wget_binary_path = config.get('web_checker', 'wget_binary_path')
email_address = config.get('web_checker', 'email_address')
smtp_host = config.get('web_checker', 'smtp_host')
debug_level = config.get('web_checker', 'debug_level')
file_log_path = 'web_checker.log'
if config.has_option('web_checker', 'file_log_path'):
file_log_path = config.get('web_checker', 'file_log_path')
header_list = {}
for header, configuration in config.items('header_list'):
......@@ -450,6 +561,14 @@ def web_checker_utility():
else:
value = configuration.splitlines()
header_list[header] = value
conditional_header_dict = {}
no_header_dict = {}
for section in config.sections():
item_list = config.items(section)
if re.compile("^header\s.*").match(section) is not None:
conditional_header_dict.setdefault(section, []).extend(item_list)
if re.compile("^no_header\s.*").match(section) is not None:
no_header_dict.setdefault(section, []).extend(item_list)
if config.has_section('erp5_extension_list'):
prohibited_file_name_list = config.get('erp5_extension_list',
'prohibited_file_name_list').splitlines()
......@@ -465,7 +584,9 @@ def web_checker_utility():
email_address,
smtp_host,
debug_level,
file_log_path)
file_log_path,
conditional_header_dict,
no_header_dict)
result = instance.start(prohibited_file_name_list=prohibited_file_name_list,
prohibited_folder_name_list=prohibited_folder_name_list)
......
product/ERP5/Document/TextDocument.py
View file @ 8383758b
......@@ -36,7 +36,7 @@ from Products.ERP5.Document.Document import Document, ConversionError, _MARKER,
from Products.ERP5.Document.File import File
from Products.ERP5Type.WebDAVSupport import TextContent
import re
from Products.ERP5.Document.Document import VALID_IMAGE_FORMAT_LIST
from Products.ERP5.Document.Document import VALID_IMAGE_FORMAT_LIST, VALID_TEXT_FORMAT_LIST
import cStringIO
from string import Template
......@@ -179,6 +179,8 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin,
self.setConversion(result, original_mime_type, **kw)
else:
mime_type, result = self.getConversion(**kw)
if format in VALID_TEXT_FORMAT_LIST:
# only textual content can be sustituted
if substitution_method_parameter_dict is None:
substitution_method_parameter_dict = {}
result = self._substituteTextContent(result, safe_substitute=safe_substitute,
......
product/ERP5OOo/tests/testDms.py
View file @ 8383758b
......@@ -2021,7 +2021,8 @@ return 1
def test_convertWebPageWithEmbeddedZODBImageToImageOnTraversal(self):
"""
Test Web Page using embedded Images into ZODB case (in its HTML body)
Test Web Page conversion to image using embedded Images into its HTML body.
Test various dumb ways to include an image (relative to instance or external ones).
"""
display= 'thumbnail'
convert_kw = {'display':display,
......@@ -2037,36 +2038,39 @@ return 1
web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
# XXX: how to simulate the case when web page contains (through reference) link to document for which based conversion failed?
# XXX: how to fix case when web page contains (through reference) link to itself (causes infinite recursion)
# images from same instance accessed by reference and wrong arguments (dispay NOT display)
# images from same instance accessed by reference and wrong conversion arguments (dispay NOT display)
# code should be more resilient
upload_file = makeFileUpload('cmyk_sample.jpg')
image = self.portal.image_module.newContent(portal_type='Image',
reference='Embedded-XXX',
version='001',
language='en')
image.setData(upload_file.read())
image.publish()
convert_kw['quality'] = 99 # to not get cached
web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
web_page_document.setTextContent('''<b> test </b><img src="Embedded-XXX?format=jpeg&amp;dispay=medium"/>''')
web_page_document.setTextContent('''<b> test </b><img src="Embedded-XXX?format=jpeg&amp;dispay=medium&amp;quality=50"/>''')
self.stepTic()
web_page_document_url = '%s/%s' %(self.portal.absolute_url(), web_page_document.getRelativeUrl())
web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
# external images
convert_kw['quality'] = 98 # to not get cached
convert_kw['quality'] = 98
web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
web_page_document.setTextContent('''<b> test </b><img src="http://www.erp5.com/images/favourite.png"/>
<img style="width: 26px; height: 26px;" src="http://www.erp5.com//images/save2.png" />
<img style="width: 26px; height: 26px;" src="http:////www.erp5.com//images/save2.png" />
<img style="width: 26px; height: 26px;" src="http://www.erp5.com/./images/save2.png" />
''')
self.stepTic()
web_page_document_url = '%s/%s' %(self.portal.absolute_url(), web_page_document.getRelativeUrl())
web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
# XXX: how to simulate the case when web page contains (through reference) link to document for which based conversion failed?
# XXX: how to fix case when web page contains (through reference) link to itself (causes infinite recursion)
def test_convertToImageOnTraversal(self):
"""
......@@ -2091,7 +2095,9 @@ return 1
image_document.edit(file=upload_file)
web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
web_page_document.setTextContent('<b> test </b>')
web_page_document.setTextContent('<b> test </b> $website_url $website_url')
# a Web Page can generate dynamic text so test is as well
web_page_document.setTextContentSubstitutionMappingMethodId('WebPage_getStandardSubstitutionMappingDict')
self.stepTic()
ooo_document_url = '%s/%s' %(self.portal.absolute_url(), ooo_document.getRelativeUrl())
......@@ -2704,8 +2710,8 @@ class TestDocumentPerformance(TestDocumentMixin):
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestDocument))
suite.addTest(unittest.makeSuite(TestDocumentWithSecurity))
suite.addTest(unittest.makeSuite(TestDocumentPerformance))
#suite.addTest(unittest.makeSuite(TestDocumentWithSecurity))
#suite.addTest(unittest.makeSuite(TestDocumentPerformance))
return suite
......
product/ERP5OOo/transforms/oood_commandtransform.py
View file @ 8383758b
......@@ -110,8 +110,10 @@ class OOOdCommandTransform(commandtransform):
url = href_attribute_list[0]
parse_result = urlparse(unquote(url))
# urlparse return a 6-tuple: scheme, netloc, path, params, query, fragment
netloc = parse_result[1]
path = parse_result[2]
if path:
if path and netloc in ('', None):
# it makes sense to include only relative to current site images not remote ones which can be taken by OOo
# OOo corrupt relative Links inside HTML content during odt conversion
# <img src="REF.TO.IMAGE" ... /> become <draw:image xlink:href="../REF.TO.IMAGE" ... />
# So remove "../" added by OOo
......@@ -119,6 +121,9 @@ class OOOdCommandTransform(commandtransform):
# in some cases like Web Page content "/../" can be contained in image URL which will break
# restrictedTraverse calls, our best guess is to remove it
path = path.replace('/../', '')
# remove occurencies of '//' or '///' in path (happens with web pages) and leave
# a traversable relative URL
path = '/'.join([x for x in path.split('/') if x.strip()!=''])
# retrieve http parameters and use them to convert image
query_parameter_string = parse_result[4]
image_parameter_dict = dict(parse_qsl(query_parameter_string))
......@@ -131,6 +136,10 @@ class OOOdCommandTransform(commandtransform):
odt_content_modified = True
content_type = image.getContentType()
format = image_parameter_dict.pop('format', None)
# convert API accepts only a certail range of arguments
for key, value in image_parameter_dict.items():
if key not in ('format', 'display', 'quality', 'resolution',):
image_parameter_dict.pop(key)
if getattr(image, 'convert', None) is not None:
# The document support conversion so perform conversion
# according given parameters
......
product/ERP5SyncML/Document/SyncMLPublication.py
View file @ 8383758b
......@@ -71,14 +71,8 @@ class SyncMLPublication(SyncMLSubscription):
"""
Reset all subscribers
"""
id_list = []
for subscriber in self.contentValues(portal_type='SyncML Subscription'):
subscriber.resetSignatureList()
id_list.append(subscriber.getId())
self.activate(activity='SQLQueue',
tag=self.getId(),
after_tag=id_list,
priority=ACTIVITY_PRIORITY).manage_delObjects(id_list)
priority=ACTIVITY_PRIORITY).manage_delObjects(ids=list(self.getObjectIds()))
security.declareProtected(Permissions.AccessContentsInformation,
'getConflictList')
......
product/ERP5SyncML/Document/SyncMLSubscription.py
View file @ 8383758b
......@@ -403,7 +403,6 @@ class SyncMLSubscription(XMLObject):
for i in xrange(0, object_list_len, MAX_OBJECTS):
current_id_list = object_id_list[i:i+MAX_OBJECTS]
self.activate(activity='SQLQueue',
tag=self.getId(),
priority=ACTIVITY_PRIORITY).manage_delObjects(current_id_list)
security.declareProtected(Permissions.AccessContentsInformation,
......@@ -449,7 +448,7 @@ class SyncMLSubscription(XMLObject):
'conflict_resolved_with_merge',
'conflict_resolved_with_client_command_winning'):
if self.getIsActivityEnabled():
signature.activate(tag=self.getId(), activity='SQLQueue',
signature.activate(activity='SQLQueue',
priority=ACTIVITY_PRIORITY).reset()
else:
signature.reset()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7