PyPI Package Versions

Seeing what wheels exist on PyPI, and what python versions they support.

Note that this doesn’t look at OS support.

Out:

/home/kevin/projects/kevbase/sphinx/examples/misc/pypi-versions.py:86: DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.
  s = pd.Series(d, name=version)
/home/kevin/projects/kevbase/sphinx/examples/misc/pypi-versions.py:81: DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.
  s = pd.Series([], name=version)

import requests
import bs4
import pandas as pd
import matplotlib.pyplot as plt
from packaging.version import parse as parse_version


def get_version_history(project):
    url = f'https://pypi.org/project/{project}'
    response = requests.get(url)
    soup = bs4.BeautifulSoup(response.content, parser='lxml')
    hist_div = soup.find(attrs={'class': 'release-timeline'})
    release_divs = hist_div.find_all(attrs={'class': 'release'})
    metadata = []
    for div in release_divs:
        version = div.find(attrs={'class': 'release__version'}).text.strip()
        version = version.replace('pre-release', '').strip()
        date = div.find(attrs={'class': 'release__version-date'}).text.strip()
        metadata.append({
            'version': version,
            'release_date': date,
        })
    return metadata


def get_files(project, release):
    url = f'https://pypi.org/project/{project}/{release}'
    response = requests.get(url)
    soup = bs4.BeautifulSoup(response.content, parser='lxml')
    files_div = soup.find(id='files')
    if files_div is None:
        return []
    file_divs = files_div.find_all('tr')
    # skip header row
    file_divs = file_divs[1:]
    metadata = []
    for div in file_divs:
        splits = div.find('th').text.split()
        file_name = splits[2]
        file_size = splits[3][1:] + ' ' + splits[4][:-1]
        cells = div.find_all('td')
        file_type = cells[0].text.replace('File type', '').strip()
        py_version = cells[1].text.replace('Python version', '').strip()
        # convert "cp37" to "3.7"
        mapping = {
            "cp26": "2.6", "cp27": "2.7", "cp32": "3.2", "cp33": "3.3", "cp34": "3.4",
            "cp35": "3.5", "cp36": "3.6", "cp37": "3.7", "cp38": "3.8", "cp39": "3.9",
            "cp310": "3.10", "cp311": "3.11", "cp312": "3.12",
        }
        py_version = mapping.get(py_version, py_version)

        metadata.append({
            'file_name': file_name,
            'file_size': file_size,
            'file_type': file_type,
            'python_version': py_version,
        })
    return metadata


projects = ['pandas', 'numpy']
fig, axes = plt.subplots(len(projects), 1, figsize=(14, 4*len(projects)))

for ax, project in zip(axes, projects):
    versions = get_version_history(project)

    version_support_info = []
    for version_info in versions:
        version = version_info['version']
        version_files = pd.DataFrame(get_files(project, version))
        if version_files.empty:
            s = pd.Series([], name=version)
            version_support_info.append(s)
            continue
        supported_python_versions = version_files['python_version'].unique()
        d = {ver: True for ver in supported_python_versions if ver != 'None'}
        s = pd.Series(d, name=version)
        version_support_info.append(s)

    support = pd.DataFrame({s.name: s for s in version_support_info})

    df = support.fillna(False).astype(int).iloc[:, ::-1]
    # sort so that 3.10 > 3.9
    df = df.sort_index(key=lambda idx: list(map(parse_version, idx)))

    ax.pcolormesh(df.columns, df.index, df, shading='auto')
    ax.set_ylabel('Python Version')
    ax.set_xlabel('Package Version')
    plt.setp(ax.get_xticklabels(), rotation=90)
    ax.set_title(project)

fig.tight_layout()
fig.show()

previous

Benford’s Law

next

Blog