Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ Unreleased.

This stabilizes the `no-distutils-patch` preview feature.

- **Use `tar-codec` for tar archives by default** ([#22094](https://github.com/astral-sh/uv/pull/22094))

Previously, uv used `astral-tokio-tar` to extract tar archives, build source distributions with `uv_build`, and read their metadata for `uv publish`. Now, uv uses `tar-codec`, which applies stricter validation when reading archives. This can cause uv to reject archives containing hard links or unsupported tar extensions that previous versions accepted. Source distributions created by `uv_build` can also have different archive bytes and hashes.

You can opt out of this behavior by setting `UV_LEGACY_TAR_BACKEND=1`.

This stabilizes the `tar-codec` preview feature.

## 0.12.x

See [changelogs/0.12.x](./changelogs/0.12.x.md)
Expand Down Expand Up @@ -71,4 +79,3 @@ See [changelogs/0.2.x](./changelogs/0.2.x.md)
See [changelogs/0.1.x](./changelogs/0.1.x.md)

<!-- prettier-ignore-end -->

6 changes: 5 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/uv-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ uv-preview = { workspace = true }
uv-pypi-types = { workspace = true }
uv-python = { workspace = true }
uv-resolver = { workspace = true }
uv-static = { workspace = true }
uv-types = { workspace = true }
uv-workspace = { workspace = true }

Expand Down
9 changes: 3 additions & 6 deletions crates/uv-bench/benches/uv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ use uv_distribution_types::Requirement;
use uv_extract::dirhash::UnhashedFile;
use uv_extract::hash::{HashReader, Hasher};
use uv_install_wheel::{InstallState, Layout, LinkMode};
use uv_preview::{MaybePreviewFeature, Preview, PreviewFeature};
use uv_preview::Preview;
use uv_pypi_types::{HashAlgorithm, Scheme};
use uv_python::PythonEnvironment;
use uv_resolver::Manifest;
use uv_static::TarBackend;

const MANY_FILES_WHEEL_FILENAME: &str = "manyfiles-0.0.0-py3-none-any.whl";
const MANY_FILES_WHEEL_FILE_COUNT: usize = 10_000;
Expand Down Expand Up @@ -234,11 +235,6 @@ fn unpack_sdist_many_files(c: &mut Criterion<WallTime>) {
.build()
.expect("Failed to create Tokio runtime");

uv_preview::set(Preview::from_feature_names(&[MaybePreviewFeature::Known(
PreviewFeature::TarCodec,
)]))
.expect("Failed to configure tar backend preview features");

c.bench_function("unpack_sdist_many_files", |b| {
b.iter_batched(
|| {
Expand All @@ -255,6 +251,7 @@ fn unpack_sdist_many_files(c: &mut Criterion<WallTime>) {
archive,
SourceDistExtension::TarGz,
extracted_sdist,
TarBackend::default(),
))
.expect("Failed to unpack sdist");
let source_tree = uv_extract::strip_component(extracted_sdist.path())
Expand Down
10 changes: 8 additions & 2 deletions crates/uv-bin-install/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ use url::Url;
use uv_client::retryable_on_request_failure;
use uv_distribution_filename::LegacySourceDistExtension;
use uv_distribution_filename::SourceDistExtension;
use uv_static::{astral_mirror_base_url, astral_mirror_url_from_env, custom_astral_mirror_url};
use uv_static::{
TarBackend, astral_mirror_base_url, astral_mirror_url_from_env, custom_astral_mirror_url,
};

use uv_cache::{Cache, CacheBucket, CacheEntry, Error as CacheError};
use uv_client::{BaseClient, RetriableError, fetch_with_url_fallback};
Expand Down Expand Up @@ -701,6 +703,7 @@ pub async fn bin_install(
retry_policy,
cache,
reporter,
TarBackend::from_env(),
)
.await
}
Expand All @@ -716,6 +719,7 @@ async fn bin_install_from_urls(
retry_policy: &ExponentialBackoff,
cache: &Cache,
reporter: &dyn Reporter,
tar_backend: TarBackend,
) -> Result<PathBuf, Error> {
let cache_entry = CacheEntry::new(
cache
Expand Down Expand Up @@ -750,6 +754,7 @@ async fn bin_install_from_urls(
format,
url,
&cache_entry,
tar_backend,
)
},
)
Expand Down Expand Up @@ -786,6 +791,7 @@ async fn download_and_unpack(
format: ArchiveFormat,
download_url: DisplaySafeUrl,
cache_entry: &CacheEntry,
tar_backend: TarBackend,
) -> Result<PathBuf, Error> {
// Create a temporary directory for extraction
let temp_dir = tempfile::tempdir_in(cache.bucket(CacheBucket::Binaries))?;
Expand Down Expand Up @@ -842,7 +848,7 @@ async fn download_and_unpack(

let id = reporter.on_download_start(binary.name(), version, size);
let mut progress_reader = ProgressReader::new(reader, id, reporter);
let (temp_dir, _) = stream::archive(&mut progress_reader, format.into(), temp_dir)
let (temp_dir, _) = stream::archive(&mut progress_reader, format.into(), temp_dir, tar_backend)
.await
.map_err(|e| Error::Extract { source: e })?;
reporter.on_download_complete(id);
Expand Down
1 change: 1 addition & 0 deletions crates/uv-build-backend/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ uv-pep508 = { workspace = true }
uv-platform-tags = { workspace = true }
uv-preview = { workspace = true }
uv-pypi-types = { workspace = true }
uv-static = { workspace = true }
uv-version = { workspace = true }
uv-toml = { workspace = true }
uv-warnings = { workspace = true }
Expand Down
89 changes: 66 additions & 23 deletions crates/uv-build-backend/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ mod tests {
use fs_err::File;
use futures_lite::future::block_on;
use indoc::indoc;
use insta::assert_snapshot;
use insta::{allow_duplicates, assert_snapshot};
use itertools::Itertools;
use regex::regex;
use sha2::Digest;
Expand All @@ -497,6 +497,7 @@ mod tests {
use uv_errors::{ErrorWithHints, Hinted};
use uv_fs::{copy_dir_all, relative_to};
use uv_preview::PreviewFeature;
use uv_static::TarBackend;

use crate::source_dist::SyncReader;

Expand Down Expand Up @@ -545,6 +546,14 @@ mod tests {
/// Run both a direct wheel build and an indirect wheel build through a source distribution,
/// while checking that directly built wheel and indirectly built wheel are the same.
fn build(source_root: &Path, dist: &Path) -> Result<BuildResults, Error> {
build_with_options(source_root, dist, TarBackend::default())
}

fn build_with_options(
source_root: &Path,
dist: &Path,
tar_backend: TarBackend,
) -> Result<BuildResults, Error> {
// Build a direct wheel, capture all its properties to compare it with the indirect wheel
// latest and remove it since it has the same filename as the indirect wheel.
let (_name, direct_wheel_list_files) = list_wheel(source_root, MOCK_UV_VERSION, false)?;
Expand All @@ -560,7 +569,8 @@ mod tests {
// TODO(konsti): This should run in the unpacked source dist tempdir, but we need to
// normalize the path.
let (_name, wheel_list_files) = list_wheel(source_root, MOCK_UV_VERSION, false)?;
let source_dist_filename = build_source_dist(source_root, dist, MOCK_UV_VERSION, false)?;
let source_dist_filename =
build_source_dist(source_root, dist, MOCK_UV_VERSION, false, tar_backend)?;
let source_dist_path = dist.join(source_dist_filename.to_string());
let source_dist_contents = sdist_contents(&source_dist_path);

Expand Down Expand Up @@ -714,10 +724,12 @@ mod tests {
/// platform-independent deterministic builds.
#[test]
fn built_by_uv_building() {
built_by_uv_building_with_backend(
&[],
"1d9ce1ce63195fbee07314c0b595ba9e063670da8d10c252c351b21e94e3f508",
);
allow_duplicates! {
built_by_uv_building_with_backend(
TarBackend::default(),
"88540014e8884fff1d6479c0c7315fcf497ba2a46f6db5c9f6e04f64a8620dcc",
);
}
}

#[test]
Expand Down Expand Up @@ -749,18 +761,17 @@ mod tests {
}

#[test]
fn built_by_uv_building_tar_codec() {
built_by_uv_building_with_backend(
&[PreviewFeature::TarCodec],
"88540014e8884fff1d6479c0c7315fcf497ba2a46f6db5c9f6e04f64a8620dcc",
);
fn built_by_uv_building_tokio_tar() {
allow_duplicates! {
built_by_uv_building_with_backend(
TarBackend::TokioTar,
"1d9ce1ce63195fbee07314c0b595ba9e063670da8d10c252c351b21e94e3f508",
);
}
}

fn built_by_uv_building_with_backend(
preview_features: &[PreviewFeature],
expected_source_dist_hash: &str,
) {
let _preview = uv_preview::test::with_features(preview_features);
fn built_by_uv_building_with_backend(tar_backend: TarBackend, expected_source_dist_hash: &str) {
let _preview = uv_preview::test::with_features(&[]);
let built_by_uv = Path::new("../../test/packages/built-by-uv");
let src = TempDir::new().unwrap();
for dir in [
Expand Down Expand Up @@ -822,7 +833,7 @@ mod tests {

// Perform both the direct and the indirect build.
let dist = TempDir::new().unwrap();
let build = build(src.path(), dist.path()).unwrap();
let build = build_with_options(src.path(), dist.path(), tar_backend).unwrap();

let source_dist_path = dist.path().join(build.source_dist_filename.to_string());
assert_eq!(
Expand Down Expand Up @@ -990,7 +1001,14 @@ mod tests {

// Build a wheel from a source distribution
let output_dir = TempDir::new().unwrap();
build_source_dist(src.path(), output_dir.path(), "0.5.15", false).unwrap();
build_source_dist(
src.path(),
output_dir.path(),
"0.5.15",
false,
TarBackend::default(),
)
.unwrap();
let sdist_tree = TempDir::new().unwrap();
let source_dist_path = output_dir.path().join("pep_pep639_license-1.0.0.tar.gz");
unpack_sdist(&source_dist_path, sdist_tree.path()).unwrap();
Expand Down Expand Up @@ -1221,7 +1239,13 @@ mod tests {
let dist = TempDir::new().unwrap();

// Source dist build should fail
let sdist_result = build_source_dist(src.path(), dist.path(), MOCK_UV_VERSION, false);
let sdist_result = build_source_dist(
src.path(),
dist.path(),
MOCK_UV_VERSION,
false,
TarBackend::default(),
);
assert!(sdist_result.is_err());

// Wheel build should fail
Expand Down Expand Up @@ -1267,7 +1291,13 @@ mod tests {
fs_err::write(&wheel_path, old_content).unwrap();

// Build should fail and delete existing files
let sdist_result = build_source_dist(src.path(), dist.path(), MOCK_UV_VERSION, false);
let sdist_result = build_source_dist(
src.path(),
dist.path(),
MOCK_UV_VERSION,
false,
TarBackend::default(),
);
assert!(sdist_result.is_err());

let wheel_result = build_wheel(src.path(), dist.path(), None, MOCK_UV_VERSION, false);
Expand Down Expand Up @@ -1323,7 +1353,14 @@ mod tests {
fs_err::write(&wheel_path, old_content).unwrap();

// Build should succeed and overwrite existing files
build_source_dist(src.path(), dist.path(), MOCK_UV_VERSION, false).unwrap();
build_source_dist(
src.path(),
dist.path(),
MOCK_UV_VERSION,
false,
TarBackend::default(),
)
.unwrap();
build_wheel(src.path(), dist.path(), None, MOCK_UV_VERSION, false).unwrap();

// Verify files were overwritten (content should be different)
Expand Down Expand Up @@ -1994,8 +2031,14 @@ mod tests {
.unwrap();

let dist = TempDir::new().unwrap();
let source_dist_filename =
build_source_dist(tmp_dir.path(), dist.path(), MOCK_UV_VERSION, false).unwrap();
let source_dist_filename = build_source_dist(
tmp_dir.path(),
dist.path(),
MOCK_UV_VERSION,
false,
TarBackend::default(),
)
.unwrap();
let source_dist_path = dist.path().join(source_dist_filename.to_string());
let contents = sdist_contents(&source_dist_path);

Expand Down
18 changes: 11 additions & 7 deletions crates/uv-build-backend/src/source_dist.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ use tracing::{debug, trace};
use uv_distribution_filename::{SourceDistExtension, SourceDistFilename};
use uv_fs::{Simplified, normalize_path};
use uv_globfilter::{GlobDirFilter, PortableGlobParser};
use uv_preview::PreviewFeature;
use uv_pypi_types::BuildKind;
use uv_static::TarBackend;
use uv_warnings::warn_user_once;
use walkdir::WalkDir;

Expand All @@ -36,6 +36,7 @@ pub fn build_source_dist(
source_dist_directory: &Path,
uv_version: &str,
show_warnings: bool,
tar_backend: TarBackend,
) -> Result<SourceDistFilename, Error> {
let pyproject_toml = PyProjectToml::parse(&source_tree.join("pyproject.toml"))?;
let filename = SourceDistFilename {
Expand All @@ -50,12 +51,15 @@ pub fn build_source_dist(
}

let temp_file = uv_fs::tempfile_in(source_dist_directory)?;
if uv_preview::is_enabled(PreviewFeature::TarCodec) {
let writer = TarCodecGzWriter::new(temp_file.as_file(), &source_dist_path);
write_source_dist(source_tree, writer, uv_version, show_warnings)?;
} else {
let writer = TokioTarGzWriter::new(temp_file.as_file(), &source_dist_path);
write_source_dist(source_tree, writer, uv_version, show_warnings)?;
match tar_backend {
TarBackend::TarCodec => {
let writer = TarCodecGzWriter::new(temp_file.as_file(), &source_dist_path);
write_source_dist(source_tree, writer, uv_version, show_warnings)?;
}
TarBackend::TokioTar => {
let writer = TokioTarGzWriter::new(temp_file.as_file(), &source_dist_path);
write_source_dist(source_tree, writer, uv_version, show_warnings)?;
}
}
temp_file
.persist(&source_dist_path)
Expand Down
Loading
Loading
⌂ 换网址