Patch #1276356: Implement new resource "urlfetch" for regrtest.

This enables even impatient people to run tests that require remote
files such as test_normalization and test_codecmaps_*.

Lib/test/regrtest.py

@@ -97,6 +97,8 @@ resources to test.  Currently only the following are defined:
 
     subprocess  Run all tests for the subprocess module.
 
+    urlfetch -  It is okay to download files required on testing.
+
 To enable all resources except one, use '-uall,-<resource>'.  For
 example, to run all the tests except for the bsddb tests, give the
 option '-uall,-bsddb'.
@@ -140,7 +142,7 @@ if sys.platform == 'darwin':
 from test import test_support
 
 RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',
-                  'decimal', 'compiler', 'subprocess')
+                  'decimal', 'compiler', 'subprocess', 'urlfetch')
 
 
 def usage(code, msg=''):
@@ -671,20 +673,12 @@ def printlist(x, width=70, indent=4):
 #     test_pep277
 #         The _ExpectedSkips constructor adds this to the set of expected
 #         skips if not os.path.supports_unicode_filenames.
-#     test_normalization
-#         Whether a skip is expected here depends on whether a large test
-#         input file has been downloaded.  test_normalization.skip_expected
-#         controls that.
 #     test_socket_ssl
 #         Controlled by test_socket_ssl.skip_expected.  Requires the network
 #         resource, and a socket module with ssl support.
 #     test_timeout
 #         Controlled by test_timeout.skip_expected.  Requires the network
 #         resource and a socket module.
-#     test_codecmaps_*
-#         Whether a skip is expected here depends on whether a large test
-#         input file has been downloaded.  test_codecmaps_*.skip_expected
-#         controls that.
 
 _expectations = {
     'win32':
@@ -1056,7 +1050,6 @@ _expectations = {
         test_macfs
         test_macostools
         test_nis
-        test_normalization
         test_ossaudiodev
         test_pep277
         test_plistlib
@@ -1108,12 +1101,8 @@ _expectations['freebsd7'] = _expectations['freebsd4']
 class _ExpectedSkips:
     def __init__(self):
         import os.path
-        from test import test_normalization
         from test import test_socket_ssl
         from test import test_timeout
-        from test import test_codecmaps_cn, test_codecmaps_jp
-        from test import test_codecmaps_kr, test_codecmaps_tw
-        from test import test_codecmaps_hk
 
         self.valid = False
         if sys.platform in _expectations:
@@ -1126,19 +1115,12 @@ class _ExpectedSkips:
             if not os.path.supports_unicode_filenames:
                 self.expected.add('test_pep277')
 
-            if test_normalization.skip_expected:
-                self.expected.add('test_normalization')
-
             if test_socket_ssl.skip_expected:
                 self.expected.add('test_socket_ssl')
 
             if test_timeout.skip_expected:
                 self.expected.add('test_timeout')
 
-            for cc in ('cn', 'jp', 'kr', 'tw', 'hk'):
-                if eval('test_codecmaps_' + cc).skip_expected:
-                    self.expected.add('test_codecmaps_' + cc)
-
             if sys.maxint == 9223372036854775807L:
                 self.expected.add('test_rgbimg')
                 self.expected.add('test_imageop')

Lib/test/test_codecmaps_cn.py

@@ -12,13 +12,11 @@ import unittest
 class TestGB2312Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gb2312'
-    mapfilename = 'EUC-CN.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-CN.TXT'
 
 class TestGBKMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gbk'
-    mapfilename = 'CP936.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/' \
                  'MICSFT/WINDOWS/CP936.TXT'
 
@@ -28,6 +26,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestGBKMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestGB2312Map, TestGBKMap)
 if __name__ == "__main__":
     test_main()

Lib/test/test_codecmaps_hk.py

@@ -12,7 +12,6 @@ import unittest
 class TestBig5HKSCSMap(test_multibytecodec_support.TestBase_Mapping,
                        unittest.TestCase):
     encoding = 'big5hkscs'
-    mapfilename = 'BIG5HKSCS.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT'
 
 def test_main():
@@ -20,6 +19,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestBig5HKSCSMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestBig5HKSCSMap)
 if __name__ == "__main__":
     test_main()

Lib/test/test_codecmaps_jp.py

@@ -12,7 +12,6 @@ import unittest
 class TestCP932Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp932'
-    mapfilename = 'CP932.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
                  'WINDOWS/CP932.TXT'
     supmaps = [
@@ -71,9 +70,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestSJISX0213Map))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestCP932Map,
-    TestEUCJPCOMPATMap, TestSJISCOMPATMap, TestEUCJISX0213Map,
-    TestSJISX0213Map)
-
 if __name__ == "__main__":
     test_main()

Lib/test/test_codecmaps_kr.py

@@ -12,7 +12,6 @@ import unittest
 class TestCP949Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp949'
-    mapfilename = 'CP949.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT' \
                  '/WINDOWS/CP949.TXT'
 
@@ -20,14 +19,12 @@ class TestCP949Map(test_multibytecodec_support.TestBase_Mapping,
 class TestEUCKRMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'euc_kr'
-    mapfilename = 'EUC-KR.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-KR.TXT'
 
 
 class TestJOHABMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'johab'
-    mapfilename = 'JOHAB.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/' \
                  'KSC/JOHAB.TXT'
     # KS X 1001 standard assigned 0x5c as WON SIGN.
@@ -44,7 +41,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestJOHABMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestCP949Map,
-    TestEUCKRMap, TestJOHABMap)
 if __name__ == "__main__":
     test_main()

Lib/test/test_codecmaps_tw.py

@@ -12,14 +12,12 @@ import unittest
 class TestBIG5Map(test_multibytecodec_support.TestBase_Mapping,
                   unittest.TestCase):
     encoding = 'big5'
-    mapfilename = 'BIG5.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/' \
                  'EASTASIA/OTHER/BIG5.TXT'
 
 class TestCP950Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp950'
-    mapfilename = 'CP950.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
                  'WINDOWS/CP950.TXT'
     pass_enctest = [
@@ -33,6 +31,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestCP950Map))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestBIG5Map, TestCP950Map)
 if __name__ == "__main__":
     test_main()

Lib/test/test_multibytecodec_support.py

@@ -163,15 +163,16 @@ class TestBase_Mapping(unittest.TestCase):
 
     def __init__(self, *args, **kw):
         unittest.TestCase.__init__(self, *args, **kw)
-        if not os.path.exists(self.mapfilename):
-            raise test_support.TestSkipped('%s not found, download from %s' %
-                    (self.mapfilename, self.mapfileurl))
+        self.open_mapping_file() # test it to report the error early
+
+    def open_mapping_file(self):
+        return test_support.open_urlresource(self.mapfileurl)
 
     def test_mapping_file(self):
         unichrs = lambda s: u''.join(map(unichr, map(eval, s.split('+'))))
         urt_wa = {}
 
-        for line in open(self.mapfilename):
+        for line in self.open_mapping_file():
             if not line:
                 break
             data = line.split('#')[0].strip().split()
@@ -217,16 +218,3 @@ def load_teststring(encoding):
     else:
         from test import cjkencodings_test
         return cjkencodings_test.teststring[encoding]
-
-def register_skip_expected(*cases):
-    for case in cases: # len(cases) must be 1 at least.
-        for path in [os.path.curdir, os.path.pardir]:
-            fn = os.path.join(path, case.mapfilename)
-            if os.path.exists(fn):
-                case.mapfilename = fn
-                break
-        else:
-            sys.modules[case.__module__].skip_expected = True
-            break
-    else:
-        sys.modules[case.__module__].skip_expected = False

Lib/test/test_normalization.py

-from test.test_support import verbose, TestFailed, TestSkipped, verify
+from test.test_support import (verbose, TestFailed, TestSkipped, verify,
+                               open_urlresource)
 import sys
 import os
 from unicodedata import normalize
 
 TESTDATAFILE = "NormalizationTest-3.2.0" + os.extsep + "txt"
-
-# This search allows using a build directory just inside the source
-# directory, and saving just one copy of the test data in the source
-# tree, rather than having a copy in each build directory.
-# There might be a better way to do this.
-
-for path in [os.path.curdir, os.path.pardir]:
-    fn = os.path.join(path, TESTDATAFILE)
-    skip_expected = not os.path.exists(fn)
-    if not skip_expected:
-        TESTDATAFILE = fn
-        break
+TESTDATAURL = "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE
 
 class RangeError:
     pass
@@ -40,12 +30,8 @@ def unistr(data):
     return u"".join([unichr(x) for x in data])
 
 def test_main():
-    if skip_expected:
-        raise TestSkipped(TESTDATAFILE + " not found, download from " +
-                    "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE)
-
     part1_data = {}
-    for line in open(TESTDATAFILE):
+    for line in open_urlresource(TESTDATAURL):
         if '#' in line:
             line = line.split('#')[0]
         line = line.strip()

Lib/test/test_support.py

@@ -237,7 +237,21 @@ def check_syntax(statement):
     else:
         print 'Missing SyntaxError: "%s"' % statement
 
+def open_urlresource(url):
+    import urllib, urlparse
+    import os.path
 
+    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
+
+    for path in [os.path.curdir, os.path.pardir]:
+        fn = os.path.join(path, filename)
+        if os.path.exists(fn):
+            return open(fn)
+
+    requires('urlfetch')
+    print >> get_original_stdout(), '\tfetching %s ...' % url
+    fn, _ = urllib.urlretrieve(url, filename)
+    return open(fn)
 
 #=======================================================================
 # Preliminary PyUNIT integration.

Misc/NEWS

@@ -633,6 +633,9 @@ Tests
 
 - In test_os, st_?time is now truncated before comparing it with ST_?TIME.
 
+- Patch #1276356: New resource "urlfetch" is implemented.  This enables
+  even impatient people to run tests that require remote files.
+
 
 Documentation
 -------------