Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
5132670
Fix library mapping in linking checks
zelosleone Jun 11, 2025
5a7584e
white lines of recipes
zelosleone Jun 11, 2025
012691e
dynamical instead of harcoded
zelosleone Jun 23, 2025
91a5277
further refining for dynamical approach
zelosleone Jun 23, 2025
c04499a
first step
zelosleone Jun 24, 2025
b495463
shallow merge
zelosleone Jul 3, 2025
9127dfd
Merge main: Add Ruby/Node.js tests while preserving cache linking tests
zelosleone Jul 3, 2025
475e8f6
merge fix
zelosleone Jul 3, 2025
cec07d0
for macos we remove paths and just use filenames
zelosleone Jul 3, 2025
1f406ad
fmt
zelosleone Jul 3, 2025
4a3e581
windows
zelosleone Jul 3, 2025
e042df3
.
zelosleone Jul 3, 2025
f1c12af
this will be a test (passes on local with vs2022)
zelosleone Jul 4, 2025
3166e2a
time to get both platforms
zelosleone Jul 4, 2025
555ff76
go
zelosleone Jul 4, 2025
992a2e7
pixi lock
zelosleone Jul 4, 2025
33b0670
some cleanup trying
zelosleone Jul 4, 2025
1de2377
lets try like this
zelosleone Jul 4, 2025
8ff0b99
LAST HOPEFULLY
zelosleone Jul 4, 2025
de0db43
cargo clippy for unix/windows file permission and test update (it pas…
zelosleone Jul 4, 2025
0b07d22
some cleanups
zelosleone Jul 7, 2025
e79868e
post process mappings with sysroot packages only
zelosleone Jul 7, 2025
aef501a
all host libraries but only sysroot from build
zelosleone Jul 7, 2025
9b57549
performance improvements with collecting library fiile names once
zelosleone Jul 7, 2025
c28c445
single lookup
zelosleone Jul 7, 2025
ebff2c5
back to ascii cases
zelosleone Jul 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,389 changes: 1,137 additions & 1,252 deletions pixi.lock

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ deploy-dev = "mike deploy --push dev devel"
[dependencies]
openssl = "3.*"
rust = ">=1.86.0,<1.87"
compilers = "1.6.0.*"
libssh2 = "1.11.0.*"
pkg-config = "0.29.2.*"
cmake = "3.27.6.*"
Expand Down Expand Up @@ -77,22 +76,28 @@ tbump = "*"
# Bump version by running `pixi run bump NEW_VERSION`
bump = "tbump --only-patch"


[target.linux-64.dependencies]
compilers = "1.6.0.*"
clang = ">=18.1.8,<19.0"
mold = ">=2.33.0,<3.0"
patchelf = ">=0.17.2,<0.18"

[target.osx-64.dependencies]
compilers = "1.6.0.*"
patchelf = ">=0.18.0,<0.19"

[target.osx-arm64.dependencies]
compilers = "1.6.0.*"
patchelf = ">=0.18.0,<0.19"

[target.win-64.dependencies]
vs2022_win-64 = ">=19.44.35207"

[target.linux-64.activation]
scripts = ["scripts/activate.sh"]
[target.osx-arm64.activation]
scripts = ["scripts/activate.sh"]

Comment on lines +80 to +100
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i seperated the compilers (that uses vs2019) from windows (manually set it to vs2022 due to bugs i encountered with earlier versions, both in CI and local)

[target.win-64.activation]
scripts = ["scripts/activate.bat"]

Expand Down
2 changes: 1 addition & 1 deletion src/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ pub async fn run_build(
.into_diagnostic()?
};

let output = output
let mut output = output
.resolve_dependencies(tool_configuration)
.await
.into_diagnostic()?;
Expand Down
58 changes: 53 additions & 5 deletions src/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,22 @@ use std::{

use fs_err as fs;
use miette::{Context, IntoDiagnostic};
use rattler_conda_types::PackageName;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};

use crate::{
env_vars,
metadata::{Output, build_reindexed_channels},
packaging::Files,
post_process::package_nature::PackageNature,
recipe::{
Jinja,
parser::{Dependency, Requirements, Source},
},
render::resolved_dependencies::{
FinalizedDependencies, install_environments, resolve_dependencies,
FinalizedDependencies, install_environments, populate_library_mappings,
resolve_dependencies,
},
source::{
copy_dir::{CopyDir, CopyOptions, copy_file},
Expand Down Expand Up @@ -62,6 +65,15 @@ pub struct Cache {
pub prefix: PathBuf,
}

/// Struct to hold post-process mappings for cache reuse
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PostProcessMappings {
/// Mapping from library names to the packages that provide them
pub library_mapping: std::collections::HashMap<String, PackageName>,
/// Mapping from package names to their PackageNature
pub package_to_nature: std::collections::HashMap<PackageName, PackageNature>,
}

impl Output {
/// Compute a cache key that contains all the information that was used to
/// build the cache, including the relevant variant information.
Expand Down Expand Up @@ -220,14 +232,50 @@ impl Output {
.into_diagnostic()
.context("failed to reindex output channel")?;

let finalized_dependencies =
let mut finalized_dependencies =
resolve_dependencies(&cache.requirements, &self, &channels, tool_configuration)
.await
.unwrap();

install_environments(&self, &finalized_dependencies, tool_configuration)
.await
.into_diagnostic()?;
install_environments(
&self.build_configuration,
&finalized_dependencies,
tool_configuration,
)
.await
.into_diagnostic()?;

// Populate library mappings after packages are installed
populate_library_mappings(
&mut finalized_dependencies,
&self.build_configuration.directories.build_prefix,
&self.build_configuration.directories.host_prefix,
)
.into_diagnostic()?;

// Write post-process-mappings.json to the cache folder
fs::create_dir_all(&cache_dir).into_diagnostic()?;
let post_process_mappings = PostProcessMappings {
library_mapping: finalized_dependencies
.host
.iter()
.chain(finalized_dependencies.build.iter())
.flat_map(|env| env.library_mapping.clone().into_iter())
.filter(|(_lib, pkg)| pkg.as_normalized().starts_with("sysroot_"))
.collect(),
package_to_nature: finalized_dependencies
.host
.iter()
.chain(finalized_dependencies.build.iter())
.flat_map(|env| env.package_nature.clone())
.collect(),
};
let mappings_path = cache_dir.join("post-process-mappings.json");
fs::write(
&mappings_path,
serde_json::to_string_pretty(&post_process_mappings).unwrap(),
)
.into_diagnostic()?;

let selector_config = self.build_configuration.selector_config();
let mut jinja = Jinja::new(selector_config.clone());
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1075,7 +1075,7 @@ pub async fn debug_recipe(
.fetch_sources(&tool_config, apply_patch_custom)
.await
.into_diagnostic()?;
let output = output
let mut output = output
.resolve_dependencies(&tool_config)
.await
.into_diagnostic()?;
Expand Down
24 changes: 24 additions & 0 deletions src/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,27 @@ impl Output {
}
Ok(())
}

/// Return a view that merges the finalized dependencies of this output
/// with those of the optional cache build.
///
/// * `run` comes from the main `finalized_dependencies` (cache runs are
/// irrelevant for the final artefact).
/// * `host` / `build` environments are taken from the main dependencies if
/// present, otherwise from the cache. If both are present we merge their
/// fields (union of `resolved`, `specs`, `library_mapping`,
/// `package_nature`).
pub fn merged_finalized_dependencies(
&self,
) -> Option<crate::render::resolved_dependencies::FinalizedDependencies> {
let primary = self.finalized_dependencies.as_ref()?;

if let Some(cache_deps) = &self.finalized_cache_dependencies {
Some(primary.merge_with(cache_deps))
} else {
Some(primary.clone())
}
}
}

impl Output {
Expand Down Expand Up @@ -827,6 +848,7 @@ mod test {
};
use rattler_digest::{Md5, Sha256, parse_digest_from_hex};
use rstest::*;
use std::collections::HashMap;
use std::str::FromStr;
use url::Url;

Expand Down Expand Up @@ -900,6 +922,8 @@ mod test {
.unwrap(),
channel: Some("test".into()),
}],
library_mapping: HashMap::new(),
package_nature: HashMap::new(),
};

// test yaml roundtrip
Expand Down
Loading
Loading