Commit 260bc0c4 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Julien Muchembled

Cache downloaded data in zc/buildout/buildout.py:_open() in memory to accelerate remote extends.

parent f01fe7df
...@@ -27,6 +27,11 @@ try: ...@@ -27,6 +27,11 @@ try:
except ImportError: except ImportError:
from collections import MutableMapping as DictMixin from collections import MutableMapping as DictMixin
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
import zc.buildout.configparser import zc.buildout.configparser
import copy import copy
import datetime import datetime
...@@ -1206,10 +1211,6 @@ class Buildout(DictMixin): ...@@ -1206,10 +1211,6 @@ class Buildout(DictMixin):
self[name] # Add to parts self[name] # Add to parts
def parse(self, data): def parse(self, data):
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
import textwrap import textwrap
sections = zc.buildout.configparser.parse( sections = zc.buildout.configparser.parse(
...@@ -1622,6 +1623,7 @@ def _default_globals(): ...@@ -1622,6 +1623,7 @@ def _default_globals():
return globals_defs return globals_defs
_open_download_cache = {}
def _open(base, filename, seen, dl_options, override, downloaded): def _open(base, filename, seen, dl_options, override, downloaded):
"""Open a configuration file and return the result as a dictionary, """Open a configuration file and return the result as a dictionary,
...@@ -1651,8 +1653,14 @@ def _open(base, filename, seen, dl_options, override, downloaded): ...@@ -1651,8 +1653,14 @@ def _open(base, filename, seen, dl_options, override, downloaded):
base = os.path.dirname(filename) base = os.path.dirname(filename)
else: else:
filename = base + '/' + filename filename = base + '/' + filename
downloaded_filename, is_temp = download(filename) data = _open_download_cache.get(filename)
fp = open(downloaded_filename) if data is None:
downloaded_filename, is_temp = download(filename)
data = open(downloaded_filename).read()
_open_download_cache[filename] = data
else:
is_temp = False
fp = StringIO(data)
base = filename[:filename.rfind('/')] base = filename[:filename.rfind('/')]
else: else:
filename = os.path.join(base, filename) filename = os.path.join(base, filename)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment