Try constructing the fake package at the beginning of the test.

setuptools/tests/test_config.py

@@ -54,6 +54,7 @@ def fake_env(
         '    return [3, 4, 5, "dev"]\n'
         '\n'
     )
+
     return package_dir, config
 
 
@@ -268,11 +269,23 @@ class TestMetadata:
 
     def test_version(self, tmpdir):
 
-        _, config = fake_env(
+        package_dir, config = fake_env(
             tmpdir,
             '[metadata]\n'
             'version = attr: fake_package.VERSION\n'
         )
+
+        sub_a = package_dir.mkdir('subpkg_a')
+        sub_a.join('__init__.py').write('')
+        sub_a.join('mod.py').write('VERSION = (2016, 11, 26)')
+
+        sub_b = package_dir.mkdir('subpkg_b')
+        sub_b.join('__init__.py').write('')
+        sub_b.join('mod.py').write(
+            'import third_party_module\n'
+            'VERSION = (2016, 11, 26)'
+        )
+
         with get_dist(tmpdir) as dist:
             assert dist.metadata.version == '1.2.3'
 
@@ -290,10 +303,6 @@ class TestMetadata:
         with get_dist(tmpdir) as dist:
             assert dist.metadata.version == '1'
 
-        sub_a = tmpdir.join('fake_package').mkdir('subpkg_a')
-        sub_a.join('__init__.py').write('')
-        sub_a.join('mod.py').write('VERSION = (2016, 11, 26)')
-
         config.write(
             '[metadata]\n'
             'version = attr: fake_package.subpkg_a.mod.VERSION\n'
@@ -301,13 +310,6 @@ class TestMetadata:
         with get_dist(tmpdir) as dist:
             assert dist.metadata.version == '2016.11.26'
 
-        sub_b = tmpdir.join('fake_package').mkdir('subpkg_b')
-        sub_b.join('__init__.py').write('')
-        sub_b.join('mod.py').write(
-            'import third_party_module\n'
-            'VERSION = (2016, 11, 26)'
-        )
-
         config.write(
             '[metadata]\n'
             'version = attr: fake_package.subpkg_b.mod.VERSION\n'