/rustc/$hash prefix is not being mapped when expected

[pnkfelix]

Spawned off of #64507

I tried this code:

% rustup default
stable-aarch64-apple-darwin (default)
% rustup component list | grep installed
cargo-aarch64-apple-darwin (installed)
clippy-aarch64-apple-darwin (installed)
rust-src (installed)
rust-std-aarch64-apple-darwin (installed)
rust-std-wasm32-unknown-unknown (installed)
rustc-aarch64-apple-darwin (installed)
rustfmt-aarch64-apple-darwin (installed)
% cat map-panic.rs
fn main() {
    let _x: Option<String> = Some(42u32).map(|_| panic!("hello world"));
}
% rustc -vV
rustc 1.64.0 (a55dd71d5 2022-09-19)
binary: rustc
commit-hash: a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52
commit-date: 2022-09-19
host: aarch64-apple-darwin
release: 1.64.0
LLVM version: 14.0.6
% rustc -g map-panic.rs
% RUST_BACKTRACE=1 ./map-panic
thread 'main' panicked at 'hello world', map-panic.rs:2:50
stack backtrace:
   0: std::panicking::begin_panic
             at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/std/src/panicking.rs:616:12
   1: map_panic::main::{{closure}}
             at ./map-panic.rs:2:50
   2: core::option::Option<T>::map
             at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/option.rs:929:29
   3: map_panic::main
             at ./map-panic.rs:2:30
   4: core::ops::function::FnOnce::call_once
             at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

I expected to see this happen: I expected the /rustc/$hash to be mapped to the local paths in the installed rust-src component, rather than being left as raw /rustc/$hash

Instead, this happened: /rustc/$hash is showing up in the output.

Meta

rustc --version --verbose:

rustc 1.64.0 (a55dd71d5 2022-09-19)
binary: rustc
commit-hash: a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52
commit-date: 2022-09-19
host: aarch64-apple-darwin
release: 1.64.0
LLVM version: 14.0.6

[pnkfelix]

see related discussion from wg-debugging triage here

[pnkfelix]

in particular, @bjorn3 pointed out that that this remapping should be happening if one has the rust-src component installed, and that the code doing the remapping should be executed here:

rust/compiler/rustc_metadata/src/rmeta/decoder.rs

Lines 1419 to 1493 in 4653c93

	// Translate the virtual `/rustc/$hash` prefix back to a real directory
	// that should hold actual sources, where possible.
	//
	// NOTE: if you update this, you might need to also update bootstrap's code for generating
	// the `rust-src` component in `Src::run` in `src/bootstrap/dist.rs`.
	let virtual_rust_source_base_dir = [
	filter(sess, option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(Path::new)),
	filter(sess, sess.opts.unstable_opts.simulate_remapped_rust_src_base.as_deref()),
	];
	let try_to_translate_virtual_to_real = |name: &mut rustc_span::FileName| {
	debug!(
	"try_to_translate_virtual_to_real(name={:?}): \
	virtual_rust_source_base_dir={:?}, real_rust_source_base_dir={:?}",
	name, virtual_rust_source_base_dir, sess.opts.real_rust_source_base_dir,
	);
	for virtual_dir in virtual_rust_source_base_dir.iter().flatten() {
	if let Some(real_dir) = &sess.opts.real_rust_source_base_dir {
	if let rustc_span::FileName::Real(old_name) = name {
	if let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } =
	old_name
	{
	if let Ok(rest) = virtual_name.strip_prefix(virtual_dir) {
	let virtual_name = virtual_name.clone();
	// The std library crates are in
	// `$sysroot/lib/rustlib/src/rust/library`, whereas other crates
	// may be in `$sysroot/lib/rustlib/src/rust/` directly. So we
	// detect crates from the std libs and handle them specially.
	const STD_LIBS: &[&str] = &[
	"core",
	"alloc",
	"std",
	"test",
	"term",
	"unwind",
	"proc_macro",
	"panic_abort",
	"panic_unwind",
	"profiler_builtins",
	"rtstartup",
	"rustc-std-workspace-core",
	"rustc-std-workspace-alloc",
	"rustc-std-workspace-std",
	"backtrace",
	];
	let is_std_lib = STD_LIBS.iter().any(|l| rest.starts_with(l));
	let new_path = if is_std_lib {
	real_dir.join("library").join(rest)
	} else {
	real_dir.join(rest)
	};
	debug!(
	"try_to_translate_virtual_to_real: `{}` -> `{}`",
	virtual_name.display(),
	new_path.display(),
	);
	let new_name = rustc_span::RealFileName::Remapped {
	local_path: Some(new_path),
	virtual_name,
	};
	*old_name = new_name;
	}
	}
	}
	}
	}
	};
	let mut import_info = self.cdata.source_map_import_info.lock();
	for _ in import_info.len()..=(source_file_index as usize) {
	import_info.push(None);

and there is corresponding code in bootstrap dist code:

rust/src/bootstrap/dist.rs

Lines 876 to 903 in 4653c93

	// A lot of tools expect the rust-src component to be entirely in this directory, so if you
	// change that (e.g. by adding another directory `lib/rustlib/src/foo` or
	// `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
	// and fix them...
	//
	// NOTE: if you update the paths here, you also should update the "virtual" path
	// translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
	let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
	let src_files = ["Cargo.lock"];
	// This is the reduced set of paths which will become the rust-src component
	// (essentially libstd and all of its path dependencies).
	copy_src_dirs(
	builder,
	&builder.src,
	&["library", "src/llvm-project/libunwind"],
	&[
	// not needed and contains symlinks which rustup currently
	// chokes on when unpacking.
	"library/backtrace/crates",
	// these are 30MB combined and aren't necessary for building
	// the standard library.
	"library/stdarch/Cargo.toml",
	"library/stdarch/crates/stdarch-verify",
	"library/stdarch/crates/intrinsic-test",
	],
	&dst_src,
	);

There's some useful debug! statements mixed in there, I bet we might be able to figure out what's going on here without too much trickery...

[lqd]

But this is currently a remapping at runtime only when using -Zsimulate-remapped-rust-src-base like in UI tests, right ? Otherwise it's optionally done at compile-time on the dist builders if they set CFG_VIRTUAL_RUST_SOURCE_BASE_DIR.

[bjorn3]

CFG_VIRTUAL_RUST_SOURCE_BASE_DIR is /rustc/$hash, right? That is what it is mapped from, not what it is mapped to.

[lqd]

Its and -Zsimulate-remapped-rust-src-base's presence seem to control (at rustc build-time and at runtime respectively) whether the remapping happens in the code you listed above (otherwise virtual_rust_source_base_dir would be "empty" and try_to_translate_virtual_to_real would be a no-op), so I may be missing how this could already remap to the local rustc-src component according to Felix's comment above ?

in particular, @bjorn3 pointed out that that this remapping should be happening if one has the rust-src component installed, and that the code doing the remapping should be executed here:

[ehuss]

I believe these are not being remapped on purpose (based on #83813 (comment)). I think currently the remapping only happens for diagnostics (and not things that end up in the binary). There's some more discussion in #85463 (and #73167 and #83813 to some degree). There's also concerns about how it can affect binary size (#82188). rust-lang/rfcs#3127 also discusses it some.

[Nemo157]

These paths are also not being remapped even when explicitly requested with --remap-path-prefix=/rustc/<hash>=...

   Compiling foo v0.1.0 (/tmp/tmp.EFCKXMFnbx/foo)
     Running `CARGO=/nix/store/xxz0yn66hiwc2wvnd9cdy453bdwp3r6h-cargo-1.75.0-nightly-2023-10-20-x86_64-unknown-linux-gnu/bin/cargo CARGO_BIN_NAME=foo CARGO_CRATE_NAME=foo CARGO_MANIFEST_DIR=/tmp/tmp.EFCKXMFnbx/foo CARGO_PKG_AUTHORS='' CARGO_PKG_DESCRIPTION='' CARGO_PKG_HOMEPAGE='' CARGO_PKG_LICENSE='' CARGO_PKG_LICENSE_FILE='' CARGO_PKG_NAME=foo CARGO_PKG_README='' CARGO_PKG_REPOSITORY='' CARGO_PKG_RUST_VERSION='' CARGO_PKG_VERSION=0.1.0 CARGO_PKG_VERSION_MAJOR=0 CARGO_PKG_VERSION_MINOR=1 CARGO_PKG_VERSION_PATCH=0 CARGO_PKG_VERSION_PRE='' CARGO_PRIMARY_PACKAGE=1 LD_LIBRARY_PATH='/tmp/tmp.EFCKXMFnbx/foo/target/debug/deps:/nix/store/7wads449pq7ppcl9fqcpp9fhnys1hf12-rust-default-1.75.0-2023-10-20/lib' rustc --crate-name foo --edition=2021 src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --diagnostic-width=201 --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 -C metadata=17c1d72b6ec80106 -C extra-filename=-17c1d72b6ec80106 --out-dir /tmp/tmp.EFCKXMFnbx/foo/target/x86_64-unknown-linux-gnu/debug/deps --target x86_64-unknown-linux-gnu -C incremental=/tmp/tmp.EFCKXMFnbx/foo/target/x86_64-unknown-linux-gnu/debug/incremental -L dependency=/tmp/tmp.EFCKXMFnbx/foo/target/x86_64-unknown-linux-gnu/debug/deps -L dependency=/tmp/tmp.EFCKXMFnbx/foo/target/debug/deps --cap-lints=warn --remap-path-prefix=/rustc/4578435e1695863d921c7763d5a0add98f8e3869=/nix/store/7wads449pq7ppcl9fqcpp9fhnys1hf12-rust-default-1.75.0-nightly-2023-10-20/lib/rustlib/src/rust -Clink-arg=-fuse-ld=mold -Ctarget-cpu=native -Zrandomize-layout`
    Finished dev [unoptimized + debuginfo] target(s) in 0.06s
     Running `CARGO=/nix/store/xxz0yn66hiwc2wvnd9cdy453bdwp3r6h-cargo-1.75.0-nightly-2023-10-20-x86_64-unknown-linux-gnu/bin/cargo CARGO_MANIFEST_DIR=/tmp/tmp.EFCKXMFnbx/foo CARGO_PKG_AUTHORS='' CARGO_PKG_DESCRIPTION='' CARGO_PKG_HOMEPAGE='' CARGO_PKG_LICENSE='' CARGO_PKG_LICENSE_FILE='' CARGO_PKG_NAME=foo CARGO_PKG_README='' CARGO_PKG_REPOSITORY='' CARGO_PKG_RUST_VERSION='' CARGO_PKG_VERSION=0.1.0 CARGO_PKG_VERSION_MAJOR=0 CARGO_PKG_VERSION_MINOR=1 CARGO_PKG_VERSION_PATCH=0 CARGO_PKG_VERSION_PRE='' LD_LIBRARY_PATH='/tmp/tmp.EFCKXMFnbx/foo/target/x86_64-unknown-linux-gnu/debug/deps:/tmp/tmp.EFCKXMFnbx/foo/target/x86_64-unknown-linux-gnu/debug:/nix/store/7wads449pq7ppcl9fqcpp9fhnys1hf12-rust-default-1.75.0-2023-10-20/lib/rustlib/x86_64-unknown-linux-gnu/lib' target/x86_64-unknown-linux-gnu/debug/foo`
thread 'main' panicked at src/main.rs:2:5:
Hello, world!
stack backtrace:
   0: rust_begin_unwind
             at /rustc/4578435e1695863d921c7763d5a0add98f8e3869/library/std/src/panicking.rs:597:5
   1: core::panicking::panic_fmt
             at /rustc/4578435e1695863d921c7763d5a0add98f8e3869/library/core/src/panicking.rs:72:14
   2: foo::main
             at ./src/main.rs:2:5
   3: core::ops::function::FnOnce::call_once
             at /rustc/4578435e1695863d921c7763d5a0add98f8e3869/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.