Add UV_PYTHON_DOWNLOADS_JSON_URL to set custom managed python sources

.github/workflows/sync-python-releases.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Sync Python Releases
         run: |
           uv run -- fetch-download-metadata.py
-          uv run -- template-download-metadata.py
+          uv run -- minify-download-metadata.py
         working-directory: ./crates/uv-python
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -35,7 +35,7 @@ jobs:
           commit-message: "Sync latest Python releases"
           add-paths: |
             crates/uv-python/download-metadata.json
-            crates/uv-python/src/downloads.inc
+            crates/uv-python/src/download-metadata-minified.json
           branch: "sync-python-releases"
           title: "Sync latest Python releases"
           body: "Automated update for Python releases."

Cargo.toml

@@ -125,6 +125,7 @@ memchr = { version = "2.7.4" }
 miette = { version = "7.2.0", features = ["fancy-no-backtrace"] }
 nanoid = { version = "0.4.0" }
 nix = { version = "0.29.0" }
+once_cell = { version = "1.20.2" }
 owo-colors = { version = "4.1.0" }
 path-slash = { version = "0.2.1" }
 pathdiff = { version = "0.2.1" }

crates/uv-python/Cargo.toml

@@ -61,6 +61,7 @@ tokio-util = { workspace = true, features = ["compat"] }
 tracing = { workspace = true }
 url = { workspace = true }
 which = { workspace = true }
+once_cell = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { workspace = true }

crates/uv-python/minify-download-metadata.py

@@ -0,0 +1,43 @@
+# /// script
+# requires-python = ">=3.12"
+# ///
+"""
+Generate minified Python version download metadata json to embed in the binary.
+
+Generates the `download-metadata-minified.json` file from the `download-metadata.json` file.
+
+Usage:
+
+    uv run -- crates/uv-python/minify-download-metadata.py
+"""
+
+import json
+from pathlib import Path
+
+CRATE_ROOT = Path(__file__).parent
+VERSION_METADATA = CRATE_ROOT / "download-metadata.json"
+TARGET = CRATE_ROOT / "src" / "download-metadata-minified.json"
+
+
+def process_json(data: dict) -> dict:
+    out_data = {}
+
+    for key, value in data.items():
+        # Exclude debug variants for now, we don't support them in the Rust side
+        if value["variant"] == "debug":
+            continue
+
+        out_data[key] = value
+
+    return out_data
+
+
+def main() -> None:
+    json_data = json.loads(Path(VERSION_METADATA).read_text())
+    json_data = process_json(json_data)
+    json_string = json.dumps(json_data, separators=(",", ":"))
+    TARGET.write_text(json_string)
+
+
+if __name__ == "__main__":
+    main()

crates/uv-python/src/downloads.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+use std::collections::HashMap;
 use std::fmt::Display;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -7,8 +9,11 @@ use std::task::{Context, Poll};
 use std::time::{Duration, SystemTime};
 
 use futures::TryStreamExt;
+use itertools::Itertools;
+use once_cell::sync::OnceCell;
 use owo_colors::OwoColorize;
 use reqwest_retry::RetryPolicy;
+use serde::Deserialize;
 use thiserror::Error;
 use tokio::io::{AsyncRead, ReadBuf};
 use tokio_util::compat::FuturesAsyncReadCompatExt;
@@ -30,6 +35,7 @@ use crate::installation::PythonInstallationKey;
 use crate::libc::LibcDetectionError;
 use crate::managed::ManagedPythonInstallation;
 use crate::platform::{self, Arch, Libc, Os};
+use crate::PythonVariant;
 use crate::{Interpreter, PythonRequest, PythonVersion, VersionRequest};
 
 #[derive(Error, Debug)]
@@ -86,9 +92,13 @@ pub enum Error {
     Mirror(&'static str, &'static str),
     #[error(transparent)]
     LibcDetection(#[from] LibcDetectionError),
+    #[error("Remote python downloads JSON is not yet supported, please use a local path (without `file://` prefix)")]
+    RemoteJSONNotSupported(),
+    #[error("The json of the python downloads is invalid: {0}")]
+    InvalidPythonDownloadsJSON(String, #[source] serde_json::Error),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct ManagedPythonDownload {
     key: PythonInstallationKey,
     url: &'static str,
@@ -245,9 +255,11 @@ impl PythonDownloadRequest {
     }
 
     /// Iterate over all [`PythonDownload`]'s that match this request.
-    pub fn iter_downloads(&self) -> impl Iterator<Item = &'static ManagedPythonDownload> + '_ {
-        ManagedPythonDownload::iter_all()
-            .filter(move |download| self.satisfied_by_download(download))
+    pub fn iter_downloads(
+        &self,
+    ) -> Result<impl Iterator<Item = &'static ManagedPythonDownload> + use<'_>, Error> {
+        Ok(ManagedPythonDownload::iter_all()?
+            .filter(move |download| self.satisfied_by_download(download)))
     }
 
     /// Whether this request is satisfied by an installation key.
@@ -445,7 +457,30 @@ impl FromStr for PythonDownloadRequest {
     }
 }
 
-include!("downloads.inc");
+const BUILTIN_PYTHON_DOWNLOADS_JSON: &str = include_str!("download-metadata-minified.json");
+static PYTHON_DOWNLOADS: OnceCell<std::borrow::Cow<'static, [ManagedPythonDownload]>> =
+    OnceCell::new();
+
+#[derive(Debug, Deserialize, Clone)]
+struct JsonPythonDownload {
+    name: String,
+    arch: JsonArch,
+    os: String,
+    libc: String,
+    major: u8,
+    minor: u8,
+    patch: u8,
+    prerelease: Option<String>,
+    url: String,
+    sha256: Option<String>,
+    variant: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+struct JsonArch {
+    family: String,
+    variant: Option<String>,
+}
 
 #[derive(Debug, Clone)]
 pub enum DownloadResult {
@@ -459,14 +494,40 @@ impl ManagedPythonDownload {
         request: &PythonDownloadRequest,
     ) -> Result<&'static ManagedPythonDownload, Error> {
         request
-            .iter_downloads()
+            .iter_downloads()?
             .next()
             .ok_or(Error::NoDownloadFound(request.clone()))
     }
 
     /// Iterate over all [`ManagedPythonDownload`]s.
-    pub fn iter_all() -> impl Iterator<Item = &'static ManagedPythonDownload> {
-        PYTHON_DOWNLOADS.iter()
+    pub fn iter_all() -> Result<impl Iterator<Item = &'static ManagedPythonDownload>, Error> {
+        let runtime_source = std::env::var(EnvVars::UV_PYTHON_DOWNLOADS_JSON_URL);
+
+        let downloads = PYTHON_DOWNLOADS.get_or_try_init(|| {
+            let json_downloads: HashMap<String, JsonPythonDownload> =
+                if let Ok(json_source) = &runtime_source {
+                    if Url::parse(json_source).is_ok() {
+                        return Err(Error::RemoteJSONNotSupported());
+                    }
+
+                    let file = match fs_err::File::open(json_source) {
+                        Ok(file) => file,
+                        Err(e) => { Err(Error::Io(e)) }?,
+                    };
+
+                    serde_json::from_reader(file)
+                        .map_err(|e| Error::InvalidPythonDownloadsJSON(json_source.clone(), e))?
+                } else {
+                    serde_json::from_str(BUILTIN_PYTHON_DOWNLOADS_JSON).map_err(|e| {
+                        Error::InvalidPythonDownloadsJSON("EMBEDDED IN THE BINARY".to_string(), e)
+                    })?
+                };
+
+            let result = parse_json_downloads(json_downloads);
+            Ok(Cow::Owned(result))
+        })?;
+
+        Ok(downloads.iter())
     }
 
     pub fn url(&self) -> &'static str {
@@ -702,6 +763,115 @@ impl ManagedPythonDownload {
     }
 }
 
+fn parse_json_downloads(
+    json_downloads: HashMap<String, JsonPythonDownload>,
+) -> Vec<ManagedPythonDownload> {
+    json_downloads
+        .into_iter()
+        .filter_map(|(key, entry)| {
+            let implementation = match entry.name.as_str() {
+                "cpython" => LenientImplementationName::Known(ImplementationName::CPython),
+                "pypy" => LenientImplementationName::Known(ImplementationName::PyPy),
+                _ => LenientImplementationName::Unknown(entry.name.clone()),
+            };
+
+            let arch_str = match entry.arch.family.as_str() {
+                "armv5tel" => "armv5te".to_string(),
+                // The `gc` variant of riscv64 is the common base instruction set and
+                // is the target in `python-build-standalone`
+                // See https://github.com/astral-sh/python-build-standalone/issues/504
+                "riscv64" => "riscv64gc".to_string(),
+                value => value.to_string(),
+            };
+
+            let arch_str = if let Some(variant) = entry.arch.variant {
+                format!("{arch_str}_{variant}")
+            } else {
+                arch_str
+            };
+
+            let arch = match Arch::from_str(&arch_str) {
+                Ok(arch) => arch,
+                Err(e) => {
+                    debug!("Skipping entry {key}: Invalid arch '{arch_str}' - {e}");
+                    return None;
+                }
+            };
+
+            let os = match Os::from_str(&entry.os) {
+                Ok(os) => os,
+                Err(e) => {
+                    debug!("Skipping entry {}: Invalid OS '{}' - {}", key, entry.os, e);
+                    return None;
+                }
+            };
+
+            let libc = match Libc::from_str(&entry.libc) {
+                Ok(libc) => libc,
+                Err(e) => {
+                    debug!(
+                        "Skipping entry {}: Invalid libc '{}' - {}",
+                        key, entry.libc, e
+                    );
+                    return None;
+                }
+            };
+
+            let variant = match entry
+                .variant
+                .as_deref()
+                .map(PythonVariant::from_str)
+                .transpose()
+            {
+                Ok(Some(variant)) => variant,
+                Ok(None) => PythonVariant::default(),
+                Err(()) => {
+                    debug!(
+                        "Skipping entry {key}: Unknown python variant - {}",
+                        entry.variant.unwrap_or_default()
+                    );
+                    return None;
+                }
+            };
+
+            let version_str = format!(
+                "{}.{}.{}{}",
+                entry.major,
+                entry.minor,
+                entry.patch,
+                entry.prerelease.as_deref().unwrap_or_default()
+            );
+
+            let version = match PythonVersion::from_str(&version_str) {
+                Ok(version) => version,
+                Err(e) => {
+                    debug!("Skipping entry {key}: Invalid version '{version_str}' - {e}");
+                    return None;
+                }
+            };
+
+            let url = Box::leak(entry.url.into_boxed_str()) as &'static str;
+            let sha256 = entry
+                .sha256
+                .map(|s| Box::leak(s.into_boxed_str()) as &'static str);
+
+            Some(ManagedPythonDownload {
+                key: PythonInstallationKey::new_from_version(
+                    implementation,
+                    &version,
+                    os,
+                    arch,
+                    libc,
+                    variant,
+                ),
+                url,
+                sha256,
+            })
+        })
+        .sorted_by(|a, b| Ord::cmp(&b.key, &a.key))
+        .collect()
+}
+
 impl Error {
     pub(crate) fn from_reqwest(url: Url, err: reqwest::Error) -> Self {
         Self::NetworkError(url, WrappedReqwestError::from(err))

crates/uv-python/src/installation.rs

@@ -294,7 +294,7 @@ impl PythonInstallationKey {
         }
     }
 
-    fn new_from_version(
+    pub fn new_from_version(
         implementation: LenientImplementationName,
         version: &PythonVersion,
         os: Os,
@@ -482,6 +482,6 @@ impl Ord for PythonInstallationKey {
             .then_with(|| self.os.to_string().cmp(&other.os.to_string()))
             .then_with(|| self.arch.to_string().cmp(&other.arch.to_string()))
             .then_with(|| self.libc.to_string().cmp(&other.libc.to_string()))
-            .then_with(|| self.variant.cmp(&other.variant))
+            .then_with(|| self.variant.cmp(&other.variant).reverse()) // we want Default to come first
     }
 }