Make it possible to specify the desired threshold before a requests gets saved...

Make it possible to specify the desired threshold before a requests gets saved to a temporary file instead of being read completely into memory.

@nexedi, tests do pass (I squashed commits) so unless objections today I merge it tomorrow.

https://nexedi.erp5.net/test_result_module/20180216-696E64EA

/reviewed-on nexedi/slapos!290

software/erp5/instance-erp5-input-schema.json

@@ -126,6 +126,11 @@
               "default": 1,
               "type": "integer"
             },
+            "large-file-threshold": {
+              "description": "Requests bigger than this size get saved into a temporary file instead of being read completely into memory, in bytes",
+              "default": "512KB",
+              "type": "string"
+            },
             "port-base": {
               "allOf": [{
                 "$ref": "#/definitions/tcpv4port"

software/slapos-master/instance-erp5.cfg.in

@@ -184,6 +184,7 @@ config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
+config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', '512KB')) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {% endfor -%}

stack/erp5/buildout.cfg

@@ -182,7 +182,7 @@ md5sum = 5691b7f12d22bfb8a926dfcb592f12c8
 [template-zope-conf]
 <= download-base
 filename = zope.conf.in
-md5sum = 77ab4f229a92e02603028a0bd3772edd
+md5sum = 05424c6d5bd919d07141a43c2136afd4
 
 [template-runzope-userhosts-preloaded]
 <= download-base
@@ -331,7 +331,7 @@ rendered = ${monitor-template-dummy:target}
 [template-erp5]
 <= download-base
 filename = instance-erp5.cfg.in
-md5sum = cb0a8dfc15874939ec786a8b7ac5bff1
+md5sum = 24a220caa80db0fcc9fffdf785a3ee24
 
 [template-zeo]
 <= download-base
@@ -341,7 +341,7 @@ md5sum = b0cb0ee97cddc79112a718e065806037
 [template-zope]
 <= download-base
 filename = instance-zope.cfg.in
-md5sum = 83304f1940e0d7555bd678e2cea4c738
+md5sum = 8b8b30fab6f6a5090fe8cd404bd1c616
 link-binary =
   ${aspell:location}/bin/aspell
   ${dmtx-utils:location}/bin/dmtxwrite

stack/erp5/instance-erp5.cfg.in

@@ -182,6 +182,7 @@ config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
+config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "512KB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {% endfor -%}

stack/erp5/instance-zope.cfg.in

@@ -243,6 +243,7 @@ tidstorage-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
 {% endif -%}
 promise-path = ${erp5-promise:promise-path}
 {% set thread_amount = slapparameter_dict['thread-amount'] -%}
+{% set large_file_threshold = slapparameter_dict['large-file-threshold']  -%}
 thread-amount = {{ thread_amount }}
 {% set webdav = slapparameter_dict['webdav'] -%}
 webdav = {{ dumps(webdav) }}
@@ -294,6 +295,7 @@ node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
 {% endfor -%}
 import-list = {{ dumps(list(import_set)) }}
 zodb-dict = {{ dumps(zodb_dict) }}
+large-file-threshold = {{ large_file_threshold }} 
 {% if longrequest_logger_interval > 0 -%}
 longrequest-logger-file = {{ longrequest_logger_base_path ~ name ~ ".log" }}
 longrequest-logger-timeout = {{ longrequest_logger_timeout }}

stack/erp5/zope.conf.in

@@ -49,6 +49,11 @@ products {{ parameter_dict['instance-products'] }}
   interval {{ parameter_dict['longrequest-logger-interval'] }}
 </product-config>
 
+{% endif -%}
+
+{% if 'large-file-threshold' in parameter_dict -%}
+large-file-threshold {{ parameter_dict['large-file-threshold'] }}
+
 {% endif -%}
 {% if 'tidstorage-ip' in parameter_dict -%}
 <product-config TIDStorage>