Repair longstanding damage introduced by the AST merge:

- Get rid of pgen.c and tokenizer_pgen.c.

- Include tokenizer.c.

All the -uall tests pass on my WinXP VC7.1 box now.

PCbuild/pythoncore.vcproj

@@ -2485,9 +2485,6 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
-		<File
-			RelativePath="..\Parser\pgen.c">
-		</File>
 		<File
 			RelativePath="..\Modules\posixmodule.c">
 			<FileConfiguration
@@ -3227,7 +3224,31 @@
 			</FileConfiguration>
 		</File>
 		<File
-			RelativePath="..\Parser\tokenizer_pgen.c">
+			RelativePath="..\Parser\tokenizer.c">
+			<FileConfiguration
+				Name="Release|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="Debug|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="0"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="ReleaseItanium|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
 		</File>
 		<File
 			RelativePath="..\Python\traceback.c">