- Merge the mj-http_range_support-branch HTTP Range functionality.

- Update the CHANGES.txt file.

ZServer/FTPServer.py

@@ -369,6 +369,9 @@ class zope_ftp_channel(ftp_channel):
         response=make_response(self, self.retr_completion, line[1])
         self._response_producers = response.stdout._producers
         request=FTPRequest(line[1],'RETR',self,response)
+        # Support download restarts if possible.
+        if self.restart_position > 0:
+            request.environ['HTTP_RANGE'] = 'bytes=%d-' % self.restart_position
         handle(self.module,request,response) 
 
     def retr_completion(self, file, response):        

doc/CHANGES.txt

@@ -42,6 +42,11 @@ Zope changes
 
         - StructuredText now takes care of your locale settings. 
 
+        - Image and File objects now support the HTTP Range and If-Range
+          headers, enabeling partial downloads of such objects. This can be
+          used by clients to restart a broken download. Downloads of these
+          objects through FTP can also be restarted.
+
     Zope 2.3.0 beta 3
 
       Features Added

lib/python/OFS/tests/__init__.py

+# This helps debugging.

lib/python/ZPublisher/HTTPRangeSupport.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+"""HTTP Range support utilities.
+
+The RFC 2616 specification defines the 'Range' and 'If-Range' headers for
+enabeling partial download of published resources. This module provides a
+flag-interface and some support functions for implementing this functionality.
+
+For an implementation example, see the File class in OFS/Image.py.
+"""
+
+__version__='$Revision'[11:-2]
+
+import re, string, sys
+import Interface
+
+WHITESPACE = re.compile('\s*', re.MULTILINE)
+
+def parseRange(header):
+    """RFC 2616 (HTTP 1.1) Range header parsing.
+
+    Convert a range header to a list of slice indexes, returned as (start, end)
+    tuples. If no end was given, end is None. Note that the RFC specifies the
+    end offset to be inclusive, we return python convention indexes, where the
+    end is exclusive. Syntactically incorrect headers are to be ignored, so if
+    we encounter one we return None.
+
+    """
+
+    ranges = []
+    add = ranges.append
+
+    # First, clean out *all* whitespace. This is slightly more tolerant
+    # than the spec asks for, but hey, it makes this function much easier.
+    header = WHITESPACE.sub('', header)
+
+    # A range header only can specify a byte range
+    try: spec, sets = string.split(header, '=')
+    except ValueError: return None
+    if spec != 'bytes':
+        return None
+
+    # The sets are delimited by commas.
+    sets = string.split(sets, ',')
+    # Filter out empty values, things like ',,' are allowed in the spec
+    sets = filter(None, sets)
+    # We need at least one set
+    if not sets:
+        return None
+
+    for set in sets:
+        try: start, end = string.split(set, '-')
+        except ValueError: return None
+
+        # Catch empty sets
+        if not start and not end:
+            return None
+
+        # Convert to integers or None (which will raise errors if
+        # non-integers were used (which is what we want)).
+        try:
+            if start == '': start = None
+            else: start = int(start)
+            if end == '': end = None
+            else: end = int(end)
+        except ValueError:
+            return None
+
+        # Special case: No start means the suffix format was used, which
+        # means the end value is actually a negative start value.
+        # Convert this by making it absolute.
+        # A -0 range is converted to sys.maxint, which will result in a
+        # Unsatisfiable response if no other ranges can by satisfied either.
+        if start is None:
+            start, end = -end, None
+            if not start:
+                start = sys.maxint
+        elif end is not None:
+            end = end + 1 # Make the end of the range exclusive
+
+        if end is not None and end <= start:
+            return None
+
+        # And store
+        add((start, end))
+
+    return ranges
+
+def optimizeRanges(ranges, size):
+    """Optimize Range sets, given those sets and the length of the resource.
+
+    Optimisation is done by first expanding relative start values and open
+    ends, then sorting and combining overlapping or adjacent ranges. We also
+    remove unsatisfiable ranges (where the start lies beyond the size of the
+    resource).
+
+    """
+
+    expanded = []
+    add = expanded.append
+    for start, end in ranges:
+        if start < 0:
+            start = size + start
+        end = end or size
+        if end > size: end = size
+        # Only use satisfiable ranges
+        if start < size:
+            add((start, end))
+
+    ranges = expanded
+    ranges.sort()
+    ranges.reverse()
+    optimized = []
+    add = optimized.append
+    start, end = ranges.pop()
+    
+    while ranges:
+        nextstart, nextend = ranges.pop()
+        # If the next range overlaps or is adjacent
+        if nextstart <= end:
+            # If it falls within the current range, discard
+            if nextend <= end:
+                continue
+            
+            # Overlap, adjust end
+            end = nextend
+        else:
+            add((start, end))
+            start, end = nextstart, nextend
+
+    # Add the remaining optimized range
+    add((start, end))
+    
+    return optimized
+
+class HTTPRangeInterface(Interface.Base):
+    """Objects implementing this Interface support the HTTP Range header.
+
+    Objects implementing support for the HTTP Range header will return partial
+    content as specified in RFC 2616. Note that the'If-Range' header must
+    either be implemented correctly or result in a normal '200 OK' response at
+    all times.
+
+    This interface specifies no methods, as this functionality can either be
+    implemented in the index_html or __call__ methods of a published object.
+
+    """

lib/python/ZPublisher/tests/__init__.py

+# This helps debugging.

lib/python/ZPublisher/tests/testHTTPRangeSupport.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+import sys
+sys.path.insert(0, '.')
+try:
+    from ZPublisher.HTTPRangeSupport import parseRange, optimizeRanges
+except ImportError:
+    sys.path[0]='../..'
+    from ZPublisher.HTTPRangeSupport import parseRange, optimizeRanges
+
+import unittest
+
+class TestRangeHeaderParse(unittest.TestCase):
+
+    # Utility methods
+    def expectNone(self, header):
+        result = parseRange(header)
+        self.failUnless(result is None, 'Expected None, got %s' % `result`)
+    
+    def expectSets(self, header, sets):
+        result = parseRange(header)
+        self.failUnless(result == sets,
+            'Expected %s, got %s' % (`sets`, `result`))
+
+    # Syntactically incorrect headers
+    def testGarbage(self):
+        self.expectNone('kjahskjhdfkgkjbnbb ehgdk dsahg wlkjew lew\n =lkdskue')
+
+    def testIllegalSpec(self):
+        self.expectNone('notbytes=0-1000')
+
+    def testNoSets(self):
+        self.expectNone('bytes=')
+
+    def testEmptySets(self):
+        self.expectNone('bytes=,,,')
+
+    def testIllegalRange(self):
+        self.expectNone('bytes=foo-bar')
+
+    def testAlmostIntegers(self):
+        self.expectNone('bytes=1.0-2.0')
+
+    def testEndLowerThanStart(self):
+        self.expectNone('bytes=5-4')
+
+    # Correct headers
+    def testSimpleRange(self):
+        self.expectSets('bytes=2-20', [(2, 21)])
+
+    def testSimpleRangeAndEmpty(self):
+        self.expectSets('bytes=,2-20,', [(2, 21)])
+
+    def testSuffixRange(self):
+        self.expectSets('bytes=-100', [(-100, None)])
+
+    def testOpenEnded(self):
+        self.expectSets('bytes=100-', [(100, None)])
+       
+    def testStartEqualsEnd(self):
+        self.expectSets('bytes=100-100', [(100, 101)])
+       
+    def testMultiple(self):
+        self.expectSets('bytes=-100,,1-2,20-', 
+            [(-100, None), (1, 3), (20, None)])
+
+    def testFirstByte(self):
+        self.expectSets('bytes=0-0', [(0, 1)])
+
+    def testNegativeZero(self):
+        self.expectSets('bytes=-0', [(sys.maxint, None)])
+
+
+class TestOptimizeRanges(unittest.TestCase):
+
+    def expectSets(self, sets, size, expect):
+        result = optimizeRanges(sets, size)
+        self.failUnless(result == expect,
+            'Expected %s, got %s' % (`expect`, `result`))
+
+    def testExpandOpenEnd(self):
+        self.expectSets([(1, 2), (5, None)], 50, [(1, 2), (5, 50)])
+
+    def testMakeAbsolute(self):
+        self.expectSets([(1, 2), (-5, None)], 50, [(1, 2), (45, 50)])
+
+    def testNoOverlapInOrder(self):
+        self.expectSets([(1, 5), (1000, 2000), (3000, None)], 5000,
+            [(1, 5), (1000, 2000), (3000, 5000)])
+
+    def testNoOverlapOutOfOrder(self):
+        self.expectSets([(1000, 2000), (3000, None), (1, 5)], 5000,
+            [(1, 5), (1000, 2000), (3000, 5000)])
+       
+    def testOverlapInOrder(self):
+        self.expectSets([(1, 10), (8, 20), (25, None)], 5000,
+            [(1, 20), (25, 5000)])
+
+    def testOverlapOutOfOrder(self):
+        self.expectSets([(25, 50), (8, None), (1, 10)], 5000,
+            [(1, 5000)])
+
+    def testAdjacentInOrder(self):
+        self.expectSets([(1, 10), (10, 20), (25, 50)], 5000,
+            [(1, 20), (25, 50)])
+
+    def testAdjacentOutOfOrder(self):
+        self.expectSets([(-5, None), (40, 45)], 50, [(40, 50)])
+
+    def testOverLapAndOverflow(self):
+        # Note that one endpoint lies beyond the end.
+        self.expectSets([(-5, None), (40, 100)], 50, [(40, 50)])
+
+    def testRemoveUnsatisfiable(self):
+        self.expectSets([(sys.maxint, None), (10, 20)], 50, [(10, 20)])
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestRangeHeaderParse, 'test'))
+    suite.addTest(unittest.makeSuite(TestOptimizeRanges, 'test'))
+    return suite
+
+def main():
+    unittest.TextTestRunner().run(test_suite())
+
+def debug():
+    test_suite().debug()
+
+def pdebug():
+    import pdb
+    pdb.run('debug()')
+   
+if __name__=='__main__':
+    if len(sys.argv) > 1:
+        globals()[sys.argv[1]]()
+    else:
+        main()
+

lib/python/ZServer/FTPServer.py

@@ -369,6 +369,9 @@ class zope_ftp_channel(ftp_channel):
         response=make_response(self, self.retr_completion, line[1])
         self._response_producers = response.stdout._producers
         request=FTPRequest(line[1],'RETR',self,response)
+        # Support download restarts if possible.
+        if self.restart_position > 0:
+            request.environ['HTTP_RANGE'] = 'bytes=%d-' % self.restart_position
         handle(self.module,request,response) 
 
     def retr_completion(self, file, response):        

lib/python/webdav/Resource.py

@@ -85,7 +85,7 @@
 
 """WebDAV support - resource objects."""
 
-__version__='$Revision: 1.42 $'[11:-2]
+__version__='$Revision: 1.43 $'[11:-2]
 
 import sys, os, string, mimetypes, davcmds, ExtensionClass, Lockable
 from common import absattr, aq_base, urlfix, rfc1123_date, tokenFinder, urlbase
@@ -94,6 +94,7 @@ from urllib import quote, unquote
 from AccessControl import getSecurityManager
 from WriteLockInterface import WriteLockInterface
 import Globals, time
+from ZPublisher.HTTPRangeSupport import HTTPRangeInterface
 
 class Resource(ExtensionClass.Base, Lockable.LockableItem):
     """The Resource mixin class provides basic WebDAV support for
@@ -131,6 +132,12 @@ class Resource(ExtensionClass.Base, Lockable.LockableItem):
             response.setHeader('Date', rfc1123_date(), 1)
         response.setHeader('MS-Author-Via', 'DAV')
 
+        # HTTP Range support
+        if HTTPRangeInterface.isImplementedBy(self):
+            response.setHeader('Accept-Ranges', 'bytes')
+        else:
+            response.setHeader('Accept-Ranges', 'none')
+
     def dav__validate(self, object, methodname, REQUEST):
         msg='<strong>You are not authorized to access this resource.</strong>'
         method=None