pyenv verification source

plugins/python-build/scripts/.gitignore

@@ -0,0 +1 @@
+/venv

plugins/python-build/scripts/README.md

@@ -0,0 +1,22 @@
+# Scripts for updating python-build
+
+Install dependencies with `pip install -r requirements.txt`.
+
+## add_miniconda.py
+
+```_add_miniconda
+usage: add_miniconda.py [-h] [-d] [-v]
+
+Script to add non-"latest" miniconda releases. Written for python 3.7. Checks
+the miniconda download archives for new versions, then writes a build script
+for any which do not exist locally, saving it to plugins/python-
+build/share/python-build. Ignores releases below 4.3.30. Also ignores sub-
+patch releases if that major.minor.patch already exists, but otherwise, takes
+the latest sub-patch release for given OS/arch. Assumes all miniconda3
+releases < 4.7 default to python 3.6, and anything else 3.7.
+
+optional arguments:
+  -h, --help     show this help message and exit
+  -d, --dry-run  Do not write scripts, just report them to stdout
+  -v, --verbose  Increase verbosity of logging
+```

plugins/python-build/scripts/add_cpython.py

@@ -0,0 +1,632 @@
+#!/usr/bin/env python3
+"""Script to add CPython releases.
+
+Checks the CPython download archives for new versions,
+then writes a build script for any which do not exist locally,
+saving it to plugins/python-build/share/python-build.
+
+"""
+import argparse
+import dataclasses
+import hashlib
+import io
+import itertools
+import logging
+import operator
+import os.path
+import pathlib
+import pprint
+import re
+import subprocess
+import sys
+import typing
+import urllib.parse
+
+import jc
+import more_itertools
+import packaging.version
+import requests
+import requests_html
+import sortedcontainers
+import tqdm
+
+#CI uses exit code 1 as a signal that no new version is found
+#so have to produce a different exit code on an exception
+def _excepthook(type,value,traceback):
+    logging.error("Unhandled exception occured",exc_info=(type,value,traceback))
+    sys.exit(2)
+sys.excepthook = _excepthook
+
+logger = logging.getLogger(__name__)
+
+CUTOFF_VERSION=packaging.version.Version('3.10')
+EXCLUDED_VERSIONS= {
+}
+
+here = pathlib.Path(__file__).resolve()
+OUT_DIR: pathlib.Path = here.parent.parent / "share" / "python-build"
+
+T_THUNK=\
+'''export PYTHON_BUILD_FREE_THREADING=1
+source "${BASH_SOURCE[0]%t}"
+'''
+
+
+def adapt_script(version: packaging.version.Version,
+                 previous_version: packaging.version.Version) -> typing.Union[pathlib.Path, None]:
+
+    previous_version_path = OUT_DIR.joinpath(str(previous_version))
+
+    with previous_version_path.open("r", encoding='utf-8') as f:
+        script = f.readlines()
+    result = io.StringIO()
+    for line in script:
+        if m:=re.match(r'\s*install_package\s+"(?P<package>Python-\S+)"\s+'
+                       r'"(?P<url>\S+)"\s+.*\s+verify_py(?P<verify_py_suffix>\d+)\s+.*$',
+                       line):
+            existing_url_path = urllib.parse.urlparse(m.group('url')).path
+            try:
+                matched_download = more_itertools.one(
+                    item for item in VersionDirectory.available[version].downloads
+                    if existing_url_path.endswith(item.extension))
+            except ValueError:
+                logger.error(f'Cannot match existing URL path\'s {existing_url_path} extension '
+                             f'to available downloads {VersionDirectory.available[version].downloads}')
+                return
+            new_package_name, new_package_url = matched_download.package_name, matched_download.url
+            new_package_hash = Url.sha256_url(new_package_url, VersionDirectory.session)
+
+            verify_py_suffix = str(version.major)+str(version.minor)
+
+            line = Re.sub_groups(m,
+                                 package=new_package_name,
+                                 url=new_package_url+'#'+new_package_hash,
+                                 verify_py_suffix=verify_py_suffix)
+
+        elif m:=re.match(r'\s*install_package\s+"(?P<package>openssl-\S+)"\s+'
+                       r'"(?P<url>\S+)"\s.*$',
+                         line):
+            item = VersionDirectory.openssl.get_store_latest_release()
+
+            line = Re.sub_groups(m,
+                               package=item.package_name,
+                               url=item.url + '#' + item.hash)
+
+        elif m:=re.match(r'\s*install_package\s+"(?P<package>readline-\S+)"\s+'
+                       r'"(?P<url>\S+)"\s.*$',
+                         line):
+            item = VersionDirectory.readline.get_store_latest_release()
+
+            line = Re.sub_groups(m,
+                                 package=item.package_name,
+                                 url=item.url + '#' + item.hash)
+
+        result.write(line)
+
+    result_path = OUT_DIR.joinpath(str(version))
+    logger.info(f"Writing {result_path}")
+    result_path.write_text(result.getvalue(), encoding='utf-8')
+    result.close()
+
+    return result_path
+
+def add_version(version: packaging.version.Version):
+
+    previous_version = VersionDirectory.existing.pick_previous_version(version).version
+
+    is_prerelease_upgrade = previous_version.major==version.major\
+            and previous_version.minor==version.minor\
+            and previous_version.micro==version.micro
+
+    logger.info(f"Adding {version} based on {previous_version}"
+                + (" (prerelease upgrade)" if is_prerelease_upgrade else ""))
+
+    VersionDirectory.available.get_store_available_source_downloads(version)
+
+    new_path = adapt_script(version,
+                 previous_version)
+    if not new_path:
+        return False
+    VersionDirectory.existing.append(_CPythonExistingScriptInfo(version,str(new_path)))
+
+    cleanup_prerelease_upgrade(is_prerelease_upgrade, previous_version, version)
+
+    handle_t_thunks(version, previous_version, is_prerelease_upgrade)
+
+    print(version)
+    return True
+
+
+def cleanup_prerelease_upgrade(
+        is_prerelease_upgrade: bool,
+        previous_version: packaging.version.Version,
+        new_version: packaging.version.Version)\
+        -> None:
+    if not is_prerelease_upgrade:
+        return
+
+    previous_version_filename = str(previous_version)
+    new_version_filename = str(new_version)
+    new_version_path = OUT_DIR / new_version_filename
+
+    logger.info(f'Git moving {previous_version_filename} '
+                f'to {new_version_filename} (preserving new data)')
+
+    data = new_version_path.read_text()
+    new_version_path.unlink()
+
+    subprocess.check_call(("git","-C",OUT_DIR,
+                           "mv",
+                           previous_version_filename,
+                           new_version_filename))
+
+    new_version_path.write_text(data)
+
+    del VersionDirectory.existing[previous_version]
+
+
+def handle_t_thunks(version, previous_version, is_prerelease_upgrade):
+    if (version.major, version.minor) < (3, 13):
+        return
+
+    # an old thunk may have older version-specific code
+    # so it's safer to write a known version-independent template
+    thunk_name = (str(version) + "t")
+    thunk_path = OUT_DIR / thunk_name
+    previous_thunk_name = str(previous_version) + "t"
+    if is_prerelease_upgrade:
+        logger.info(f"Git moving {previous_thunk_name} to {thunk_name}")
+        subprocess.check_call(("git","-C",OUT_DIR,
+                              "mv",
+                              previous_thunk_name,
+                              thunk_name))
+
+    logger.info(f"Writing {thunk_path}")
+    thunk_path.write_text(T_THUNK, encoding='utf-8')
+
+
+Arguments: argparse.Namespace
+
+def main():
+    global Arguments
+    Arguments = parse_args()
+    logging.basicConfig(level=logging.DEBUG if Arguments.verbose else logging.INFO)
+
+    cached_session=requests_html.HTMLSession()
+    global VersionDirectory
+    VersionDirectory = _VersionDirectory(cached_session)
+
+    VersionDirectory.existing.populate()
+    VersionDirectory.available.populate()
+
+    # Prereleases are placed under the same directory as the corresponding release.
+    # So until we know the release is out, its directory is a potential prerelease directory.
+    # Normally, prereleases are only made for initial releases (x.y.0) --
+    # but rarely, they may make them for other releases (e.g. 3.14.5).
+    for release in (v for v in frozenset(VersionDirectory.available.keys())     #refining changes the
+                                                                                #corresponding directory key
+                                                                                #which breaks iteration
+                                                                                #so have to iterate over a copy
+                            if v not in VersionDirectory.existing):
+        VersionDirectory.available.get_store_available_source_downloads(release, True)
+        del release
+
+    versions_to_add = sorted(VersionDirectory.available.keys() - VersionDirectory.existing.keys())
+
+    logger.info("Versions to add:\n"+pprint.pformat(versions_to_add))
+    result = False
+    for version_to_add in versions_to_add:
+        result = add_version(version_to_add) or result
+    return int(not result)
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-d", "--dry-run", action="store_true",
+        help="Do not write scripts, just report them to stdout",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", default=0,
+        help="Increase verbosity of logging",
+    )
+    parsed = parser.parse_args()
+    return parsed
+
+
+T = typing.TypeVar('T', bound=object)
+
+K = typing.TypeVar('K', bound=typing.Hashable)
+
+class KeyedList(typing.List[T], typing.Mapping[K, T]):
+    key_field: str
+    item_init: typing.Callable[..., T] = None
+
+    def __init__(self, seq: typing.Union[typing.Iterable[T], None] = None):
+        super().__init__()
+        self._map = {}
+        if seq is not None:
+            self.__iadd__(seq)
+
+    # read
+
+    def __getitem__(self, key: K) -> T:
+        return self._map[key]
+
+    def __contains__(self, key: K):
+        return key in self._map
+
+    def keys(self) -> typing.AbstractSet[K]:
+        return self._map.keys()
+
+    # write
+
+    def append(self, item: T) -> None:
+        key = self._getkey(item)
+        if key in self:
+            raise ValueError(f"Key '{key:r}' already present")
+        super().append(item)
+        self._map[key] = item
+
+    def __iadd__(self, other: typing.Iterable[T]):
+        for item in other:
+            self.append(item)
+        return self
+
+    def __delitem__(self, key: K):
+        super().remove(self[key])
+        del self._map[key]
+
+    def clear(self):
+        super().__delitem__(slice(None,None))
+        self._map.clear()
+
+    # read-write
+
+    def get_or_create(self, key: K, **kwargs):
+        try:
+            return self[key]
+        except KeyError as e:
+            if self.item_init is None:
+                raise AttributeError("'item_init' must be set to use automatic item creation") from e
+            kwargs[self.key_field] = key
+            item = self.item_init(**kwargs)
+            self.append(item)
+            return item
+
+    # info
+
+    def __repr__(self):
+        return self.__class__.__name__ + "([" + ", ".join(repr(i) for i in self) + "])"
+
+    # private
+
+    def _getkey(self, item: T) -> K:
+        return getattr(item, self.key_field)
+
+del T, K
+
+@dataclasses.dataclass(frozen=True)
+class _CPythonAvailableVersionDownloadInfo:
+    extension: str
+    package_name: str
+    url: str
+
+class _CPythonAvailableVersionDownloadsDirectory(KeyedList[_CPythonAvailableVersionDownloadInfo, str]):
+    key_field = "extension"
+
+
+@dataclasses.dataclass(frozen=True)
+class _CPythonAvailableVersionInfo:
+    version: packaging.version.Version
+    download_page_url: str
+    downloads: _CPythonAvailableVersionDownloadsDirectory = dataclasses.field(
+        default_factory=lambda:_CPythonAvailableVersionDownloadsDirectory()
+    )
+
+
+class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo, packaging.version.Version]):
+    key_field = "version"
+    _session: requests.Session
+    item_init = _CPythonAvailableVersionInfo
+
+    def __init__(self, session: requests.Session, seq=None):
+        super().__init__(seq)
+        self._session = session
+
+    def populate(self):
+        """
+        Fetch remote versions
+        """
+        logger.info("Fetching available CPython versions")
+        for name, url in DownloadPage.enum_download_entries(
+                "https://www.python.org/ftp/python/",
+                r'^(\d+.*)/$', self._session,
+                make_name= lambda m: m.group(1)
+        ):
+            v = packaging.version.Version(name)
+            if v < CUTOFF_VERSION or v in EXCLUDED_VERSIONS:
+                continue
+            logger.debug(f'Available version: {name} ({v}), {url}')
+            self.append(_CPythonAvailableVersionInfo(
+                v,
+                url
+            ))
+
+    def get_store_available_source_downloads(self, version, refine_mode=False):
+        entry = self[version]
+        if entry.downloads:
+            #already retrieved
+            return
+        additional_versions_found =\
+            CPythonAvailableVersionsDirectory(self._session) if refine_mode else None
+        exact_download_found = False
+        for name, url in DownloadPage.enum_download_entries(
+                entry.download_page_url,
+                r'Python-.*\.(tar\.xz|tgz)$',
+                self._session):
+            m = re.match(r'(?P<package>Python-(?P<version>.*))\.(?P<extension>tar\.xz|tgz)$', name)
+
+            download_version = packaging.version.Version(m.group("version"))
+            if download_version != version:
+                if not refine_mode:
+                    raise ValueError(f"Unexpectedly found a download {name} ({download_version}) "
+                                     f"for {version} at page {entry.download_page_url}")
+                entry_to_fill = additional_versions_found.get_or_create(
+                    download_version,
+                    download_page_url=entry.download_page_url
+                )
+            else:
+                exact_download_found = True
+                entry_to_fill = entry
+
+            entry_to_fill.downloads.append(_CPythonAvailableVersionDownloadInfo(
+                m.group("extension"), m.group('package'), url
+            ))
+
+        if not exact_download_found:
+            actual_version = max(additional_versions_found.keys())
+            logger.debug(f"Refining available version {version} to {actual_version}")
+            del self[version]
+
+            self.append(
+                additional_versions_found[
+                    actual_version
+                ])
+
+
+class _CPythonExistingScriptInfo(typing.NamedTuple):
+    version: packaging.version.Version
+    filename: str
+
+class CPythonExistingScriptsDirectory(KeyedList[_CPythonExistingScriptInfo, packaging.version.Version]):
+    key_field = "version"
+    _filename_pattern = r'^\d+\.\d+(?:(t?)(-\w+)|(.\d+((?:a|b|rc)\d)?(t?)))$'
+
+    def populate(self):
+        """
+        Enumerate existing installation scripts in share/python-build/ by pattern
+        """
+        logger.info(f"Enumerating existing versions in {OUT_DIR}")
+        for entry_name in (p.name for p in OUT_DIR.iterdir() if p.is_file()):
+            if (not (m := re.match(self._filename_pattern, entry_name))
+                    or m.group(1) == 't' or m.group(5) == 't'):
+                continue
+            try:
+                v = packaging.version.Version(entry_name)
+                if v < CUTOFF_VERSION:
+                    continue
+                # branch tip scrpts are different from release scripts and thus unusable as a pattern
+                if v.dev is not None:
+                    continue
+                logger.debug(f"Existing version {v}")
+
+                self.append(_CPythonExistingScriptInfo(v, entry_name))
+
+            except ValueError as e:
+                logger.error(f"Unable to parse existing version {entry_name}: {e}")
+
+    def pick_previous_version(self,
+                              version: packaging.version.Version) \
+            -> _CPythonExistingScriptInfo:
+        return max(v for v in self if v.version < version)
+
+
+class _OpenSSLVersionInfo(typing.NamedTuple):
+    version: packaging.version.Version
+    package_name: str
+    url: str
+    hash: str
+
+class OpenSSLVersionsDirectory(KeyedList[_OpenSSLVersionInfo, packaging.version.Version]):
+    key_field = "version"
+
+    def get_store_latest_release(self) \
+            -> _OpenSSLVersionInfo:
+        if self:
+            #already retrieved
+            return self[max(self.keys())]
+
+        j = requests.get("https://api.github.com/repos/openssl/openssl/releases/latest", timeout=30).json()
+        # noinspection PyTypeChecker
+        # urlparse can parse str as well as bytes
+        shasum_url = more_itertools.one(
+            asset['browser_download_url']
+            for asset in j['assets']
+            if urllib.parse.urlparse(asset['browser_download_url']).path.split('/')[-1].endswith('.sha256')
+        )
+        shasum_text = requests.get(shasum_url, timeout=30).text
+        shasum_data = jc.parse("hashsum", shasum_text, quiet=True)[0]
+        package_hash, package_filename = shasum_data["hash"], shasum_data["filename"]
+        del shasum_data, shasum_text, shasum_url
+
+        # OpenSSL Github repo has tag names "openssl-<version>" as of this writing like we need
+        # but let's not rely on that
+        # splitext doesn't work with a chained extension, it only splits off the last one
+        package_name, package_version_str = re.match(r"([^-]+-(.*?))\.\D", package_filename).groups()
+        package_version = packaging.version.Version(package_version_str)
+
+        package_url = more_itertools.one(
+            asset['browser_download_url']
+            for asset in j['assets']
+            if urllib.parse.urlparse(asset['browser_download_url']).path.split('/')[-1] == package_filename
+        )
+
+        result = _OpenSSLVersionInfo(package_version, package_name, package_url, package_hash)
+        self.append(result)
+
+        return result
+
+
+class _ReadlineVersionInfo(typing.NamedTuple):
+    version : packaging.version.Version
+    package_name : str
+    url : str
+    hash : str
+
+class ReadlineVersionsDirectory(KeyedList[_ReadlineVersionInfo, packaging.version.Version]):
+    key_field = "version"
+
+    def get_store_latest_release(self):
+        if not self:
+            self._store_latest_release()
+        return self._latest_release()
+
+    def _store_latest_release(self):
+        candidates = ReadlineVersionsDirectory()
+
+        pattern = r'(?P<package_name>readline-(?P<version>\d+(?:\.\d+)+)).tar\.gz$'
+        for name, url in DownloadPage.enum_download_entries(
+                'https://ftpmirror.gnu.org/readline/', pattern, VersionDirectory.session):
+            m = re.match(pattern, name)
+            version = packaging.version.Version(m.group('version'))
+            candidates.append(_ReadlineVersionInfo(
+                version,
+                m.group('package_name'),
+                url,
+                ""
+            ))
+        max_item = candidates._latest_release()
+        hash_ = Url.sha256_url(max_item.url, VersionDirectory.session)
+
+        permalink = 'https://ftpmirror.gnu.org/readline/' +\
+              os.path.basename(urllib.parse.urlparse(max_item.url).path)
+
+        result = _ReadlineVersionInfo(
+            max_item.version,
+            max_item.package_name,
+            permalink,
+            hash_)
+        self.append(result)
+
+        return result
+
+    def _latest_release(self):
+        return self[max(self.keys())]
+
+class _VersionDirectory:
+    def __init__(self, session):
+        self.existing = CPythonExistingScriptsDirectory()
+        self.available = CPythonAvailableVersionsDirectory(session)
+        self.openssl = OpenSSLVersionsDirectory()
+        self.readline = ReadlineVersionsDirectory()
+        self.session = session
+VersionDirectory : _VersionDirectory
+
+class DownloadPage:
+    class _DownloadPageEntry(typing.NamedTuple):
+        name: str
+        url: str
+
+    @classmethod
+    def enum_download_entries(cls, url, pattern, session=None,
+                              make_name = lambda m: m.string ) \
+            -> typing.Generator[_DownloadPageEntry, None, None]:
+        """
+        Enum download entries in a standard Apache directory page
+        (incl. CPython download page https://www.python.org/ftp/python/)
+        or a GNU mirror directory page
+        (https://ftpmirror.gnu.org/<package>/ destinations)
+        """
+        if session is None:
+            session = requests_html.HTMLSession()
+        response = session.get(url, timeout=30)
+        page = response.html
+        table = page.find("pre", first=True)
+        # some GNU mirrors format entries as a table
+        # (e.g. https://mirrors.ibiblio.org/gnu/readline/)
+        if table is None:
+            table = page.find("table", first=True)
+        links = table.find("a")
+        for link in links:
+            href = link.attrs['href']
+            # CPython entries are directories
+            name = link.text
+            # skip directory entries
+            if not (m:=re.match(pattern, name)):
+                continue
+            name = make_name(m)
+            yield cls._DownloadPageEntry(name, urllib.parse.urljoin(response.url, href))
+
+
+class Re:
+    @dataclasses.dataclass
+    class _interval:
+        group: typing.Union[int, str, None]
+        start: int
+        end: int
+    @staticmethod
+    def sub_groups(match: re.Match,
+                   /, *args: [typing.AnyStr],
+                   **kwargs: [typing.AnyStr])\
+            -> typing.AnyStr:
+        repls={i:repl for i,repl in enumerate(args) if repl is not None}
+        repls.update({n:repl for n,repl in kwargs.items() if repl is not None})
+
+        intervals: sortedcontainers.SortedList[Re._interval]=\
+            sortedcontainers.SortedKeyList(key=operator.attrgetter("start","end"))
+
+        for group_id in itertools.chain(range(1,len(match.groups())), match.groupdict().keys()):
+            if group_id not in repls:
+                continue
+            if match.start(group_id) == -1:
+                continue
+            intervals.add(Re._interval(group_id,match.start(group_id),match.end(group_id)))
+        del group_id
+
+        last_interval=Re._interval(None,0,0)
+        result=""
+        for interval in intervals:
+            if interval.start < last_interval.end:
+                raise ValueError(f"Cannot replace intersecting matches "
+                                 f"for groups {last_interval.group} and {interval.group} "
+                                 f"(position {interval.start})")
+            if interval.end == interval.start and \
+                    last_interval.start == last_interval.end == interval.start:
+                raise ValueError(f"Cannot replace consecutive zero-length matches "
+                                 f"for groups {last_interval.group} and {interval.group} "
+                                 f"(position {interval.start})")
+
+            result+=match.string[last_interval.end:interval.start]+repls[interval.group]
+            last_interval = interval
+        result+=match.string[last_interval.end:]
+
+        return result
+
+class Url:
+    @staticmethod
+    def sha256_url(url, session=None):
+        if session is None:
+            session = requests_html.HTMLSession()
+        logger.info(f"Downloading and computing hash of {url}")
+        h=hashlib.sha256()
+        r=session.get(url,stream=True,timeout=30)
+        total_bytes=int(r.headers.get('content-length',0)) or float('inf')
+        with tqdm.tqdm(total=total_bytes, unit='B', unit_scale=True, unit_divisor=1024) as t:
+            for c in r.iter_content(1024):
+                t.update(len(c))
+                h.update(c)
+        return h.hexdigest()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

plugins/python-build/scripts/add_miniconda.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+"""Script to add non-"latest" miniconda releases.
+Written for python 3.7.
+
+Checks the miniconda download archives for new versions,
+then writes a build script for any which do not exist locally,
+saving it to plugins/python-build/share/python-build.
+
+Ignores releases below 4.3.30.
+Also ignores sub-patch releases if that major.minor.patch already exists,
+but otherwise, takes the latest sub-patch release for given OS/arch.
+Assumes all miniconda3 releases < 4.7 default to python 3.6, and anything else 3.7.
+"""
+import logging
+import re
+import string
+import sys
+import textwrap
+from argparse import ArgumentParser
+from collections import defaultdict
+from dataclasses import dataclass
+from enum import Enum
+from functools import total_ordering
+from pathlib import Path
+from typing import NamedTuple, List, Optional, DefaultDict, Dict
+
+import requests_html
+
+logger = logging.getLogger(__name__)
+
+CONDA_REPO = "https://repo.anaconda.com"
+MINICONDA_REPO = CONDA_REPO + "/miniconda"
+ANACONDA_REPO = CONDA_REPO + "/archive"
+
+auto_accept_tos_fmt="""export CONDA_PLUGINS_AUTO_ACCEPT_TOS=true
+""".strip()
+
+install_script_fmt = """
+{auto_accept_tos}
+case "$(anaconda_architecture 2>/dev/null || true)" in
+{install_lines}
+* )
+  {{ echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of {tflavor} is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  }} >&2
+  exit 1
+  ;;
+esac
+""".lstrip()
+
+install_line_fmt = """
+"{os}-{arch}" )
+  install_script "{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}" "{repo}/{tflavor}{suffix}-{version_py_version}{version_str}-{os}-{arch}.sh#{md5}" "{flavor}" verify_{py_version}
+  ;;
+""".strip()
+
+here = Path(__file__).resolve()
+out_dir: Path = here.parent.parent / "share" / "python-build"
+
+
+class StrEnum(str, Enum):
+    """Enum subclass whose members are also instances of str
+    and directly comparable to strings. str type is forced at declaration.
+
+    Adapted from https://github.com/kissgyorgy/enum34-custom/blob/dbc89596761c970398701d26c6a5bbcfcf70f548/enum_custom.py#L100
+    (MIT license)
+    """
+
+    def __new__(cls, *args):
+        for arg in args:
+            if not isinstance(arg, str):
+                raise TypeError("Not text %s:" % arg)
+
+        return super(StrEnum, cls).__new__(cls, *args)
+
+    def __str__(self):
+        return str(self.value)
+
+
+class SupportedOS(StrEnum):
+    LINUX = "Linux"
+    MACOSX = "MacOSX"
+
+
+class SupportedArch(StrEnum):
+    AARCH64 = "aarch64"
+    ARM64 = "arm64"
+    PPC64LE = "ppc64le"
+    S390X = "s390x"
+    X86_64 = "x86_64"
+    X86 = "x86"
+
+
+class Flavor(StrEnum):
+    ANACONDA = "anaconda"
+    MINICONDA = "miniconda"
+
+
+class TFlavor(StrEnum):
+    ANACONDA = "Anaconda"
+    MINICONDA = "Miniconda"
+
+
+class Suffix(StrEnum):
+    TWO = "2"
+    THREE = "3"
+    NONE = ""
+
+
+PyVersion = None
+class PyVersionMeta(type):
+    def __getattr__(self, name):
+        """Generate PyVersion.PYXXX on demand to future-proof it"""
+        if PyVersion is not None:
+            return PyVersion(name.lower())
+        return super(PyVersionMeta,self).__getattr__(self, name)
+
+
+@dataclass(frozen=True)
+class PyVersion(metaclass=PyVersionMeta):
+    major: str
+    minor: str
+
+    def __init__(self, value):
+        (major, minor) = re.match(r"py(\d)(\d+)", value).groups()
+        object.__setattr__(self, "major", major)
+        object.__setattr__(self, "minor", minor)
+
+    @property
+    def value(self):
+        return f"py{self.major}{self.minor}"
+
+    def version(self):
+        return f"{self.major}.{self.minor}"
+
+    def version_info(self):
+        return (self.major, self.minor)
+
+    def __str__(self):
+        return self.value
+
+
+@total_ordering
+class VersionStr(str):
+    def info(self):
+        return tuple(int(n) for n in self.replace("-", ".").split("."))
+
+    def __eq__(self, other):
+        return str(self) == str(other)
+
+    def __lt__(self, other):
+        if isinstance(other, VersionStr):
+            return self.info() < other.info()
+        raise ValueError("VersionStr can only be compared to other VersionStr")
+
+    @classmethod
+    def from_info(cls, version_info):
+        return VersionStr(".".join(str(n) for n in version_info))
+
+    def __hash__(self):
+        return hash(str(self))
+
+
+class CondaVersion(NamedTuple):
+    flavor: Flavor
+    suffix: Suffix
+    version_str: VersionStr
+    py_version: Optional[PyVersion]
+
+    @classmethod
+    def from_str(cls, s):
+        """
+        Convert a string of the form "miniconda_n-ver" or "miniconda_n-py_ver-ver" to a :class:`CondaVersion` object.
+        """
+        miniconda_n, _, remainder = s.partition("-")
+        suffix = miniconda_n[-1]
+        if suffix in string.digits:
+            flavor = miniconda_n[:-1]
+        else:
+            flavor = miniconda_n
+            suffix = ""
+
+        components = remainder.split("-")
+        if flavor == Flavor.MINICONDA and len(components) >= 2:
+            py_ver, *ver_parts = components
+            py_ver = PyVersion(f"py{py_ver.replace('.', '')}")
+            ver = "-".join(ver_parts)
+        else:
+            ver = "-".join(components)
+            py_ver = None
+
+        return CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver)
+
+    def to_filename(self):
+        if self.py_version:
+            return f"{self.flavor}{self.suffix}-{self.py_version.version()}-{self.version_str}"
+        else:
+            return f"{self.flavor}{self.suffix}-{self.version_str}"
+
+    def default_py_version(self):
+        """
+        :class:`PyVersion` of Python used with this Miniconda version
+        """
+        if self.py_version:
+            return self.py_version
+        elif self.suffix == Suffix.TWO:
+            return PyVersion.PY27
+
+        v = self.version_str.info()
+        if self.flavor == "miniconda":
+            # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-python.html
+            if v < (4, 7):
+                return PyVersion.PY36
+            if v < (4, 8):
+                return PyVersion.PY37
+            else:
+                # since 4.8, Miniconda specifies versions explicitly in the file name
+                raise ValueError("Miniconda 4.8+ is supposed to specify a Python version explicitly")
+        if self.flavor == "anaconda":
+            # https://www.anaconda.com/docs/tools/anaconda-org/release-notes
+            if v >= (2025,6):
+                return PyVersion.PY313
+            if v >= (2024,6):
+                return PyVersion.PY312
+            if v >= (2023,7):
+                return PyVersion.PY311
+            if v >= (2023,3):
+                return PyVersion.PY310
+            if v >= (2021,11):
+                return PyVersion.PY39
+            if v >= (2020,7):
+                return PyVersion.PY38
+            if v >= (2020,2):
+                return PyVersion.PY37
+            if v >= (5,3,0):
+                return PyVersion.PY37
+            return PyVersion.PY36
+
+        raise ValueError(self.flavor)
+        
+    def requires_tos_accept(self):
+        """
+        requires to accept TOS for installation
+        """
+        return self.flavor == Flavor.MINICONDA \
+            and self.version_str.info() >= (25,)
+
+
+class CondaSpec(NamedTuple):
+    tflavor: TFlavor
+    version: CondaVersion
+    os: SupportedOS
+    arch: SupportedArch
+    md5: str
+    repo: str
+    py_version: Optional[PyVersion] = None
+
+    @classmethod
+    def from_filestem(cls, stem, md5, repo, py_version=None):
+        # The `*vers` captures the new trailing `-1` in some file names (a build number?)
+        # so they can be processed properly.
+        miniconda_n, *vers, os, arch = stem.split("-")
+        ver = "-".join(vers)
+        suffix = miniconda_n[-1]
+        if suffix in string.digits:
+            tflavor = miniconda_n[:-1]
+        else:
+            tflavor = miniconda_n
+            suffix = ""
+        flavor = tflavor.lower()
+
+        if ver.startswith("py"):
+            py_ver, ver = ver.split("_", maxsplit=1)
+            py_ver = PyVersion(py_ver)
+        else:
+            py_ver = None
+        spec = CondaSpec(
+            TFlavor(tflavor),
+            CondaVersion(Flavor(flavor), Suffix(suffix), VersionStr(ver), py_ver),
+            SupportedOS(os),
+            SupportedArch(arch),
+            md5,
+            repo,
+            py_ver
+        )
+        if py_version is None and py_ver is None and ver != "latest":
+            spec = spec.with_py_version(spec.version.default_py_version())
+        return spec
+
+    def to_install_lines(self):
+        """
+        Installation command for this version of Miniconda for use in a Pyenv installation script
+        """
+        return install_line_fmt.format(
+            tflavor=self.tflavor,
+            flavor=self.version.flavor,
+            repo=self.repo,
+            suffix=self.version.suffix,
+            version_str=self.version.version_str,
+            version_py_version=f"{self.version.py_version}_" if self.version.py_version else "",
+            os=self.os,
+            arch=self.arch,
+            md5=self.md5,
+            py_version=self.py_version,
+        )
+
+    def with_py_version(self, py_version: PyVersion):
+        return CondaSpec(*self[:-1], py_version=py_version)
+
+
+def make_script(specs: List[CondaSpec]):
+    install_lines = [s.to_install_lines() for s in specs]
+    return install_script_fmt.format(
+        install_lines="\n".join(install_lines),
+        tflavor=specs[0].tflavor,
+        auto_accept_tos = auto_accept_tos_fmt if specs[0].version.requires_tos_accept() else ""
+    ).lstrip()
+
+
+def get_existing_condas(name):
+    """
+    Enumerate existing Miniconda installation scripts in share/python-build/ except rolling releases.
+
+    :returns: A generator of :class:`CondaVersion` objects.
+    """
+    logger.info("Getting known %(name)s versions",locals())
+    for p in out_dir.iterdir():
+        entry_name = p.name
+        if not p.is_file() or not entry_name.startswith(name):
+            continue
+        try:
+            v = CondaVersion.from_str(entry_name)
+            if v.version_str != "latest":
+                logger.debug("Found existing %(name)s version %(v)s", locals())
+                yield v
+        except ValueError as e:
+            logger.error("Unable to parse existing version %s: %s", entry_name, e)
+
+
+def get_available_condas(name, repo):
+    """
+    Fetch remote miniconda versions.
+
+    :returns: A generator of :class:`CondaSpec` objects for each release available for download
+    except rolling releases.
+    """
+    logger.info("Fetching remote %(name)s versions",locals())
+    session = requests_html.HTMLSession()
+    response = session.get(repo)
+    page: requests_html.HTML = response.html
+    table = page.find("table", first=True)
+    rows = table.find("tr")[1:]
+    for row in rows:
+        f, size, date, md5 = row.find("td")
+        fname = f.text
+        md5 = md5.text
+
+        if not fname.endswith(".sh"):
+            continue
+        stem = fname[:-3]
+
+        try:
+            s = CondaSpec.from_filestem(stem, md5, repo)
+            if s.version.version_str != "latest":
+                logger.debug("Found remote %(name)s version %(s)s", locals())
+                yield s
+        except ValueError:
+            pass
+
+
+def key_fn(spec: CondaSpec):
+    return (
+        spec.tflavor,
+        spec.version.version_str.info(),
+        spec.version.suffix.value,
+        spec.os.value,
+        spec.arch.value,
+    )
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-d", "--dry-run", action="store_true",
+        help="Do not write scripts, just report them to stdout",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", default=0,
+        help="Increase verbosity of logging",
+    )
+    parsed = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if parsed.verbose else logging.INFO)
+
+    existing_versions = set()
+    available_specs = set()
+    for name,repo in ("miniconda",MINICONDA_REPO),("anaconda",ANACONDA_REPO):
+        existing_versions |= set(get_existing_condas(name))
+        available_specs |= set(get_available_condas(name, repo))
+
+    # version triple to triple-ified spec to raw spec
+    to_add: DefaultDict[
+        CondaVersion, Dict[CondaSpec, CondaSpec]
+    ] = defaultdict(dict)
+
+    logger.info("Checking for new versions")
+    for s in sorted(available_specs, key=key_fn):
+        key = s.version
+        vv = key.version_str.info()
+
+        reason = None
+        if key in existing_versions:
+            reason = "already exists"
+        elif key.version_str.info() <= (4, 3, 30):
+            reason = "too old"
+        elif len(key.version_str.info()) >= 4 and "-" not in key.version_str:
+            reason = "ignoring hotfix releases"
+
+        if reason:
+            logger.debug("Ignoring version %(s)s (%(reason)s)", locals())
+            continue
+
+        to_add[key][s] = s
+
+    logger.info("Writing %s scripts", len(to_add))
+    for ver, d in to_add.items():
+        specs = list(d.values())
+        fpath = out_dir / ver.to_filename()
+        script_str = make_script(specs)
+        logger.info("Writing script for %s", ver)
+        if parsed.dry_run:
+            print(f"Would write spec to {fpath}:\n" + textwrap.indent(script_str, "  "))
+        else:
+            with open(fpath, "w") as f:
+                f.write(script_str)

plugins/python-build/scripts/add_miniforge.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+'Adds the latest miniforge and mambaforge releases.'
+from pathlib import Path
+import logging
+import os
+import string
+
+import requests
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO'))
+
+MINIFORGE_REPO = 'conda-forge/miniforge'
+DISTRIBUTIONS = ['miniforge']
+DISTRIBUTIONS_PRE25  = ['miniforge', 'mambaforge']
+
+SKIPPED_RELEASES = [
+    '4.13.0-0',     #has no Mambaforge. We already generated scripts for Miniforge
+    '22.11.1-0',    #MacOS packages are broken (have broken dep tarballs, downloading them fails with 403)
+    '22.11.1-1',    #MacOS packages are broken (have broken dep tarballs, downloading them fails with 403)
+    '22.11.1-2',    #MacOS packages are broken (have broken dep tarballs, downloading them fails with 403)
+    '25.3.0-0',     #marked as prerelease, no Linux version
+    '25.11.0-0',    #regression reported in constructor, re-released as 25.11.0-1 with hotfix bumping to constructor>=3.14 (was >=3.12, 3.13 implicit)
+]
+
+install_script_fmt = """
+case "$(anaconda_architecture 2>/dev/null || true)" in
+{install_lines}
+* )
+  {{ echo
+    colorize 1 "ERROR"
+    echo ": The binary distribution of {flavor} is not available for $(anaconda_architecture 2>/dev/null || true)."
+    echo
+  }} >&2
+  exit 1
+  ;;
+esac
+""".lstrip()
+
+install_line_fmt = """
+"{os}-{arch}" )
+  install_script "{filename}" "{url}#{sha}" "miniconda" verify_py{py_version}
+  ;;
+""".strip()
+
+here = Path(__file__).resolve()
+out_dir: Path = here.parent.parent / "share" / "python-build"
+
+def download_sha(url):
+    logger.info(f'Downloading SHA file {url}')
+    tup = tuple(reversed(requests.get(url).text.replace('./', '').rstrip().split()))
+    logger.debug(f'Got {tup}')
+    return tup
+
+def create_spec(filename, sha, url):
+    flavor_with_suffix, version, subversion, os, arch = filename.replace('.sh', '').split('-')
+    suffix = flavor_with_suffix[-1]
+
+    if suffix in string.digits:
+        flavor = flavor_with_suffix[:-1]
+    else:
+        flavor = flavor_with_suffix
+
+    spec = {
+        'filename': filename,
+        'sha': sha,
+        'url': url,
+        'py_version': py_version(version),
+        'flavor': flavor,
+        'os': os,
+        'arch': arch,
+        'installer_filename': f'{flavor_with_suffix.lower()}-{version}-{subversion}',
+    }
+
+    logger.debug(f'Created spec {spec}')
+
+    return spec
+
+def version_tuple(version):
+    return tuple(int(part) for part in version.split('-')[0].split("."))
+
+def py_version(version):
+    """Suffix for `verify_pyXXX` to call in the generated build script"""
+    version_tuple_ = version_tuple(version)
+    # current version: mentioned under https://github.com/conda-forge/miniforge?tab=readme-ov-file#requirements-and-installers
+    # transition points:
+    # https://github.com/conda-forge/miniforge/blame/main/Miniforge3/construct.yaml
+    # look for "- python <version>" in non-pypy branch and which tag the commit is first in
+    if version_tuple_ >= (26,1):
+        # https://github.com/conda-forge/miniforge/commit/0016367731e52c67234d6d0e7e6a24c6bf7673e4
+        return "313"
+    if version_tuple_ >= (24,5):
+        # yes, they jumped from 3.10 directly to 3.12
+        # https://github.com/conda-forge/miniforge/commit/bddad0baf22b37cfe079e47fd1680fdfb2183590
+        return "312"
+    if version_tuple_ >= (4,14):
+        return "310"
+    raise ValueError("Bundled Python version unknown for release `%s'"%version)
+
+def supported(filename):
+    return ('pypy' not in filename) and ('Windows' not in filename) and (not filename.endswith('.pkg'))
+
+def add_version(release, distributions):
+    tag_name = release['tag_name']
+    download_urls = { f['name']: f['browser_download_url'] for f in release['assets'] }
+    # can assume that sha files are named similar to release files so can also check supported(on their names)
+    shas = dict([download_sha(url) for (name, url) in download_urls.items()
+        if name.endswith('.sha256') and supported(os.path.splitext(name)[0]) and tag_name in name])
+    specs = [create_spec(filename, sha, download_urls[filename]) for (filename, sha) in shas.items() if supported(filename)]
+    
+
+    for distribution in distributions:
+        distribution_specs = [spec for spec in specs if distribution in spec['flavor'].lower()]
+        count = len(distribution_specs)
+
+        if count > 0:
+            output_file = out_dir / distribution_specs[0]['installer_filename']
+
+            logger.info(f'Writing {count} specs for {distribution} to {output_file}')
+
+            script_str = install_script_fmt.format(
+                install_lines="\n".join([install_line_fmt.format_map(s) for s in distribution_specs]),
+                flavor=distribution_specs[0]['flavor'],
+            )
+
+            with open(output_file, 'w') as f:
+                f.write(script_str)
+        else:
+            logger.info(f'Did not find specs for {distribution}')
+
+
+def main():
+    for release in requests.get(f'https://api.github.com/repos/{MINIFORGE_REPO}/releases').json():
+        version = release['tag_name']
+
+        if version in SKIPPED_RELEASES:
+            continue
+
+        logger.info(f'Looking for {version} in {out_dir}')
+
+        # mambaforge is retired https://github.com/conda-forge/miniforge/releases/tag/24.11.2-0
+        if version_tuple(version) >= (24, 11, 2):
+            distributions = DISTRIBUTIONS
+        else:
+            distributions = DISTRIBUTIONS_PRE25
+
+        if any(not list(out_dir.glob(f'{distribution}*-{version}')) for distribution in distributions):
+            logger.info(f'Downloading {version}')
+            add_version(release, distributions)
+
+if __name__ == '__main__':
+    main()

plugins/python-build/scripts/requirements.txt

@@ -0,0 +1,9 @@
+more_itertools
+requests-html
+fake_useragent<2
+lxml[html_clean]
+packaging
+requests
+sortedcontainers
+tqdm
+jc @ git+https://github.com/native-api/jc@haslib_mode