Adding script from idg serpro. Ref #432.

scripts/upload-old-releases-as-zip.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import errno
+import glob
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+import tempfile
+import urllib
+import urllib2
+
+from distutils.version import LooseVersion
+
+OK = '\033[92m'
+FAIL = '\033[91m'
+END = '\033[0m'
+DISTRIBUTION = "setuptools"
+
+
+class SetuptoolsOldReleasesWithoutZip(object):
+    """docstring for SetuptoolsOldReleases"""
+
+    def __init__(self):
+        super(SetuptoolsOldReleasesWithoutZip, self).__init__()
+        self.dirpath = tempfile.mkdtemp(prefix=DISTRIBUTION)
+        print "Downloading %s releases..." % DISTRIBUTION
+        print "All releases will be downloaded to %s" % self.dirpath
+        self.data_json_setuptools = self.get_json_data(DISTRIBUTION)
+        self.valid_releases_numbers = sorted([
+            release
+            for release in self.data_json_setuptools['releases']
+            # This condition is motivated by 13.0 release, which
+            # comes as "13.0": [], in the json
+            if self.data_json_setuptools['releases'][release]
+        ], key=LooseVersion)
+        self.total_downloaded_ok = 0
+
+    def get_json_data(self, package_name):
+        """
+        "releases": {
+            "0.7.2": [
+                {
+                    "has_sig": false,
+                    "upload_time": "2013-06-09T16:10:00",
+                    "comment_text": "",
+                    "python_version": "source",
+                    "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.2.tar.gz",  # NOQA
+                    "md5_digest": "de44cd90f8a1c713d6c2bff67bbca65d",
+                    "downloads": 159014,
+                    "filename": "setuptools-0.7.2.tar.gz",
+                    "packagetype": "sdist",
+                    "size": 633077
+                }
+            ],
+            "0.7.3": [
+                {
+                    "has_sig": false,
+                    "upload_time": "2013-06-18T21:08:56",
+                    "comment_text": "",
+                    "python_version": "source",
+                    "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.3.tar.gz",  # NOQA
+                    "md5_digest": "c854adacbf9067d330a847f06f7a8eba",
+                    "downloads": 30594,
+                    "filename": "setuptools-0.7.3.tar.gz",
+                    "packagetype": "sdist",
+                    "size": 751152
+                }
+            ],
+            "12.3": [
+                {
+                    "has_sig": false,
+                    "upload_time": "2015-02-26T19:15:51",
+                    "comment_text": "",
+                    "python_version": "3.4",
+                    "url": "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-12.3-py2.py3-none-any.whl",  # NOQA
+                    "md5_digest": "31f51a38497a70efadf5ce8d4c2211ab",
+                    "downloads": 288451,
+                    "filename": "setuptools-12.3-py2.py3-none-any.whl",
+                    "packagetype": "bdist_wheel",
+                    "size": 501904
+                },
+                {
+                    "has_sig": false,
+                    "upload_time": "2015-02-26T19:15:43",
+                    "comment_text": "",
+                    "python_version": "source",
+                    "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.tar.gz",  # NOQA
+                    "md5_digest": "67614b6d560fa4f240e99cd553ec7f32",
+                    "downloads": 110109,
+                    "filename": "setuptools-12.3.tar.gz",
+                    "packagetype": "sdist",
+                    "size": 635025
+                },
+                {
+                    "has_sig": false,
+                    "upload_time": "2015-02-26T19:15:47",
+                    "comment_text": "",
+                    "python_version": "source",
+                    "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.zip",  # NOQA
+                    "md5_digest": "abc799e7db6e7281535bf342bfc41a12",
+                    "downloads": 67539,
+                    "filename": "setuptools-12.3.zip",
+                    "packagetype": "sdist",
+                    "size": 678783
+                }
+            ],
+        """
+        url = "https://pypi.python.org/pypi/%s/json" % (package_name,)
+        data = json.load(urllib2.urlopen(urllib2.Request(url)))
+
+        # Mainly for debug.
+        json_filename = "%s/%s.json" % (self.dirpath, DISTRIBUTION)
+        with open(json_filename, 'w') as outfile:
+            json.dump(
+                data,
+                outfile,
+                sort_keys=True,
+                indent=4,
+                separators=(',', ': '),
+            )
+
+        return data
+
+    def get_setuptools_releases_without_zip_counterpart(self):
+        # Get set(all_valid_releases) - set(releases_with_zip), so now we have
+        # the releases without zip.
+        return set(self.valid_releases_numbers) - set([
+            release
+            for release in self.valid_releases_numbers
+                for same_version_release_dict in self.data_json_setuptools['releases'][release]  # NOQA
+                if 'zip' in same_version_release_dict['filename']
+       ])
+
+    def download_setuptools_releases_without_zip_counterpart(self):
+        try:
+            releases_without_zip = self.get_setuptools_releases_without_zip_counterpart()  # NOQA
+            failed_md5_releases = []
+            # This is a "strange" loop, going through all releases and
+            # testing only the release I need to download, but I thought it
+            # would be mouch more readable than trying to iterate through
+            # releases I need and get into traverse hell values inside dicts
+            # inside dicts of the json to get the distribution's url to
+            # download.
+            for release in self.valid_releases_numbers:
+                if release in releases_without_zip:
+                    for same_version_release_dict in self.data_json_setuptools['releases'][release]:  # NOQA
+                        if 'tar.gz' in same_version_release_dict['filename']:
+                            print "Downloading %s..." % release
+                            local_file = '%s/%s' % (
+                                self.dirpath,
+                                same_version_release_dict["filename"]
+                            )
+                            urllib.urlretrieve(
+                                same_version_release_dict["url"],
+                                local_file
+                            )
+                            targz = open(local_file, 'rb').read()
+                            hexdigest = hashlib.md5(targz).hexdigest()
+                            if (hexdigest != same_version_release_dict['md5_digest']):  # NOQA
+                                print FAIL + "FAIL: md5 for %s didn't match!" % release + END  # NOQA
+                                failed_md5_releases.append(release)
+                            else:
+                                self.total_downloaded_ok += 1
+            print 'Total releases without zip: %s' % len(releases_without_zip)
+            print 'Total downloaded: %s' % self.total_downloaded_ok
+            if failed_md5_releases:
+                raise(Exception(
+                    FAIL + (
+                        "FAIL: these releases %s failed the md5 check!" %
+                        ','.join(failed_md5_releases)
+                    ) + END
+                ))
+            elif self.total_downloaded_ok != len(releases_without_zip):
+                raise(Exception(
+                    FAIL + (
+                        "FAIL: Unknown error occured. Please check the logs."
+                    ) + END
+                ))
+            else:
+                print OK + "All releases downloaded and md5 checked." + END
+
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+    def convert_targz_to_zip(self):
+        print "Converting the tar.gz to zip..."
+        files = glob.glob('%s/*.tar.gz' % self.dirpath)
+        total_converted = 0
+        for targz in sorted(files, key=LooseVersion):
+            # Extract and remove tar.
+            tar = tarfile.open(targz)
+            tar.extractall(path=self.dirpath)
+            tar.close()
+            os.remove(targz)
+
+            # Zip the extracted tar.
+            setuptools_folder_path = targz.replace('.tar.gz', '')
+            setuptools_folder_name = setuptools_folder_path.split("/")[-1]
+            print setuptools_folder_name
+            shutil.make_archive(
+                setuptools_folder_path,
+                'zip',
+                self.dirpath,
+                setuptools_folder_name
+            )
+            # Exclude extracted tar folder.
+            shutil.rmtree(setuptools_folder_path.replace('.zip', ''))
+            total_converted += 1
+        print 'Total converted: %s' % total_converted
+        if self.total_downloaded_ok != total_converted:
+            raise(Exception(
+                FAIL + (
+                    "FAIL: Total number of downloaded releases is different"
+                    " from converted ones. Please check the logs."
+                ) + END
+            ))
+        print "Done with the tar.gz->zip. Check folder %s." % main.dirpath
+
+    def upload_zips_to_pypi(self):
+        print 'Uploading to pypi...'
+        zips = sorted(glob.glob('%s/*.zip' % self.dirpath), key=LooseVersion)
+        for zips in glob.glob('%s/*.zip' % self.dirpath):
+            # Put the twine upload code here
+            pass
+
+
+main = SetuptoolsOldReleasesWithoutZip()
+main.download_setuptools_releases_without_zip_counterpart()
+main.convert_targz_to_zip()
+main.upload_zips_to_pypi()