Skip to content

Commit fbe8224

Browse files
committed
Auto merge of #153826 - Zoxc:horde-maps, r=<try>
Use `horde`'s `SyncTable` for default query caches and `CtxtInterners`
2 parents 9e973d8 + 99cc3e8 commit fbe8224

11 files changed

Lines changed: 217 additions & 17 deletions

File tree

Cargo.lock

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,6 +1709,15 @@ dependencies = [
17091709
"windows-sys 0.61.2",
17101710
]
17111711

1712+
[[package]]
1713+
name = "horde"
1714+
version = "0.1.2"
1715+
source = "registry+https://github.com/rust-lang/crates.io-index"
1716+
checksum = "f3d7c629771c2e116e71d8e7bbc6e6e0450a8817766a7230bb0c98f81311df34"
1717+
dependencies = [
1718+
"parking_lot",
1719+
]
1720+
17121721
[[package]]
17131722
name = "html-checker"
17141723
version = "0.1.0"
@@ -3803,6 +3812,7 @@ dependencies = [
38033812
"elsa",
38043813
"ena",
38053814
"hashbrown 0.16.1",
3815+
"horde",
38063816
"indexmap",
38073817
"jobserver",
38083818
"libc",

compiler/rustc_data_structures/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ bitflags = "2.4.1"
1010
either = "1.0"
1111
elsa = "1.11.0"
1212
ena = "0.14.4"
13+
horde = { version = "0.1.2", features = ["nightly"] }
1314
indexmap = "2.12.1"
1415
jobserver_crate = { version = "0.1.28", package = "jobserver" }
1516
measureme = "12.0.1"
@@ -33,7 +34,7 @@ tracing = "0.1"
3334
[dependencies.hashbrown]
3435
version = "0.16.1"
3536
default-features = false
36-
features = ["nightly"] # for may_dangle
37+
features = ["nightly"] # for may_dangle
3738

3839
[target.'cfg(windows)'.dependencies.windows]
3940
version = "0.61.0"

compiler/rustc_data_structures/src/sync.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
use std::collections::HashMap;
2626
use std::hash::{BuildHasher, Hash};
2727

28+
pub use horde::collect;
2829
pub use parking_lot::{
2930
MappedRwLockReadGuard as MappedReadGuard, MappedRwLockWriteGuard as MappedWriteGuard,
3031
RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard,
@@ -39,13 +40,15 @@ pub use self::parallel::{
3940
broadcast, par_fns, par_for_each_in, par_join, par_map, parallel_guard, spawn,
4041
try_par_for_each_in,
4142
};
43+
pub use self::sync_table::{LockedWrite, Read, SyncTable};
4244
pub use self::vec::{AppendOnlyIndexVec, AppendOnlyVec};
4345
pub use self::worker_local::{Registry, WorkerLocal};
4446
pub use crate::marker::*;
4547

4648
mod freeze;
4749
mod lock;
4850
mod parallel;
51+
mod sync_table;
4952
mod vec;
5053
mod worker_local;
5154

compiler/rustc_data_structures/src/sync/lock.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ impl<T> Lock<T> {
112112
self.data.get_mut()
113113
}
114114

115+
#[inline(always)]
116+
pub fn mode(&self) -> Mode {
117+
self.mode
118+
}
119+
115120
#[inline(always)]
116121
pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
117122
let mode = self.mode;
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
use std::borrow::Borrow;
2+
use std::hash::{BuildHasher, Hash, Hasher};
3+
use std::ops::{Deref, DerefMut};
4+
5+
use horde::collect::{Pin, pin};
6+
pub use horde::sync_table::Read;
7+
use horde::sync_table::Write;
8+
use rustc_hash::FxBuildHasher;
9+
10+
use crate::sharded::IntoPointer;
11+
use crate::sync::{DynSync, Lock, LockGuard, Mode};
12+
13+
pub struct SyncTable<K, V> {
14+
// We use this lock to protect `table` instead of the internal mutex in `horde::SyncTable`
15+
// as it's faster when synchronization is disabled.
16+
lock: Lock<()>,
17+
18+
table: horde::SyncTable<K, V, FxBuildHasher>,
19+
}
20+
21+
// Memory reclamation can move elements to other threads for dropping,
22+
// so we require `Sync` instead of `DynSync` here
23+
unsafe impl<K: Sync, V: Sync> DynSync for SyncTable<K, V> where FxBuildHasher: Sync {}
24+
25+
impl<K, V> Default for SyncTable<K, V> {
26+
fn default() -> Self {
27+
Self { lock: Lock::default(), table: horde::SyncTable::default() }
28+
}
29+
}
30+
31+
impl<K, V> SyncTable<K, V> {
32+
/// Creates a [Read] handle from a pinned region.
33+
///
34+
/// Use [horde::collect::pin] to get a `Pin` instance.
35+
#[inline]
36+
pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, K, V, FxBuildHasher> {
37+
self.table.read(pin)
38+
}
39+
40+
/// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes.
41+
#[inline]
42+
pub fn lock(&self) -> LockedWrite<'_, K, V> {
43+
LockedWrite {
44+
_guard: self.lock.lock(),
45+
table: {
46+
// SAFETY: We ensure there's only 1 writer at a time using our own lock
47+
unsafe { self.table.unsafe_write() }
48+
},
49+
}
50+
}
51+
52+
/// Hashes a key with the table's hasher.
53+
#[inline]
54+
pub fn hash_key<Q>(&self, key: &Q) -> u64
55+
where
56+
K: Borrow<Q>,
57+
Q: ?Sized + Hash,
58+
{
59+
self.table.hash_key::<Q>(key)
60+
}
61+
62+
pub fn len(&self) -> usize {
63+
pin(|pin| self.read(pin).len())
64+
}
65+
66+
pub fn with_capacity(cap: usize) -> Self {
67+
Self { lock: Lock::new(()), table: horde::SyncTable::new_with(FxBuildHasher, cap) }
68+
}
69+
}
70+
71+
/// A handle to a [SyncTable] with write access protected by a lock.
72+
pub struct LockedWrite<'a, K, V> {
73+
table: Write<'a, K, V, FxBuildHasher>,
74+
_guard: LockGuard<'a, ()>,
75+
}
76+
77+
impl<'a, K, V> Deref for LockedWrite<'a, K, V> {
78+
type Target = Write<'a, K, V, FxBuildHasher>;
79+
80+
#[inline]
81+
fn deref(&self) -> &Self::Target {
82+
&self.table
83+
}
84+
}
85+
86+
impl<'a, K, V> DerefMut for LockedWrite<'a, K, V> {
87+
#[inline]
88+
fn deref_mut(&mut self) -> &mut Self::Target {
89+
&mut self.table
90+
}
91+
}
92+
93+
impl<K: Eq + Hash + Copy + Send> SyncTable<K, ()> {
94+
pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool
95+
where
96+
K: IntoPointer,
97+
{
98+
pin(|pin| {
99+
let mut state = FxBuildHasher.build_hasher();
100+
value.hash(&mut state);
101+
let hash = state.finish();
102+
let value = value.into_pointer();
103+
self.read(pin).get_from_hash(hash, |entry| entry.into_pointer() == value).is_some()
104+
})
105+
}
106+
107+
#[inline]
108+
pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
109+
where
110+
K: Borrow<Q>,
111+
Q: Hash + Eq,
112+
{
113+
pin(|pin| {
114+
let hash = self.hash_key(value);
115+
116+
let entry = self.read(pin).get(value, Some(hash));
117+
if let Some(entry) = entry {
118+
return *entry.0;
119+
}
120+
121+
let mut write = self.lock();
122+
123+
if self.lock.mode() == Mode::Sync {
124+
let entry = self.read(pin).get(value, Some(hash));
125+
if let Some(entry) = entry {
126+
return *entry.0;
127+
}
128+
}
129+
130+
let result = make();
131+
132+
write.insert_new(result, (), Some(hash));
133+
134+
result
135+
})
136+
}
137+
138+
#[inline]
139+
pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K
140+
where
141+
K: Borrow<Q>,
142+
Q: Hash + Eq,
143+
{
144+
pin(|pin| {
145+
let hash = self.hash_key(&value);
146+
147+
let entry = self.read(pin).get(&value, Some(hash));
148+
if let Some(entry) = entry {
149+
return *entry.0;
150+
}
151+
152+
let mut write = self.lock();
153+
154+
if self.lock.mode() == Mode::Sync {
155+
let entry = self.read(pin).get(&value, Some(hash));
156+
if let Some(entry) = entry {
157+
return *entry.0;
158+
}
159+
}
160+
161+
let result = make(value);
162+
163+
write.insert_new(result, (), Some(hash));
164+
165+
result
166+
})
167+
}
168+
}

compiler/rustc_interface/src/util.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::CodegenBackend;
1414
use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig};
1515
use rustc_data_structures::fx::FxIndexMap;
1616
use rustc_data_structures::jobserver::Proxy;
17-
use rustc_data_structures::sync;
17+
use rustc_data_structures::sync::{self, collect};
1818
use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib};
1919
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
2020
use rustc_middle::ty::{CurrentGcx, TyCtxt};
@@ -216,7 +216,10 @@ pub(crate) fn run_in_thread_pool_with_globals<
216216
let builder = rustc_thread_pool::ThreadPoolBuilder::new()
217217
.thread_name(|_| "rustc".to_string())
218218
.acquire_thread_handler(move || proxy_.acquire_thread())
219-
.release_thread_handler(move || proxy__.release_thread())
219+
.release_thread_handler(move || {
220+
collect::release();
221+
proxy__.release_thread()
222+
})
220223
.num_threads(threads)
221224
.deadlock_handler(move || {
222225
// On deadlock, creates a new thread and forwards information in thread

compiler/rustc_middle/src/query/caches.rs

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use std::sync::OnceLock;
22

3-
use rustc_data_structures::sharded::ShardedHashMap;
3+
use rustc_data_structures::sync::SyncTable;
4+
use rustc_data_structures::sync::collect::pin;
45
pub use rustc_data_structures::vec_cache::VecCache;
56
use rustc_hir::def_id::LOCAL_CRATE;
67
use rustc_index::Idx;
@@ -41,7 +42,7 @@ pub trait QueryCache: Sized {
4142
/// In-memory cache for queries whose keys aren't suitable for any of the
4243
/// more specialized kinds of cache. Backed by a sharded hashmap.
4344
pub struct DefaultCache<K, V> {
44-
cache: ShardedHashMap<K, (V, DepNodeIndex)>,
45+
cache: SyncTable<K, (V, DepNodeIndex)>,
4546
}
4647

4748
impl<K, V> Default for DefaultCache<K, V> {
@@ -53,33 +54,34 @@ impl<K, V> Default for DefaultCache<K, V> {
5354
impl<K, V> QueryCache for DefaultCache<K, V>
5455
where
5556
K: QueryKey,
56-
V: Copy,
57+
V: Copy + Send,
5758
{
5859
type Key = K;
5960
type Value = V;
6061

6162
#[inline(always)]
6263
fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
63-
self.cache.get(key)
64+
pin(|pin| {
65+
let result = self.cache.read(pin).get(key, None);
66+
if let Some((_, value)) = result { Some(*value) } else { None }
67+
})
6468
}
6569

6670
#[inline]
6771
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
68-
// We may be overwriting another value. This is all right, since the dep-graph
69-
// will check that the value fingerprint matches.
70-
self.cache.insert(key, (value, index));
72+
self.cache.lock().insert_new(key, (value, index), None);
7173
}
7274

7375
fn for_each(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
74-
for shard in self.cache.lock_shards() {
75-
for (k, v) in shard.iter() {
76+
pin(|pin| {
77+
for (k, v) in self.cache.read(pin).iter() {
7678
f(k, &v.0, v.1);
7779
}
78-
}
80+
})
7981
}
8082

8183
fn len(&self) -> usize {
82-
self.cache.len()
84+
pin(|pin| self.cache.read(pin).len())
8385
}
8486
}
8587

@@ -142,7 +144,7 @@ impl<V> Default for DefIdCache<V> {
142144

143145
impl<V> QueryCache for DefIdCache<V>
144146
where
145-
V: Copy,
147+
V: Copy + Send,
146148
{
147149
type Key = DefId;
148150
type Value = V;

compiler/rustc_middle/src/query/job.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use std::num::NonZero;
44
use std::sync::Arc;
55

66
use parking_lot::{Condvar, Mutex};
7+
use rustc_data_structures::sync::collect;
78
use rustc_span::Span;
89

910
use crate::query::plumbing::CycleError;
@@ -112,6 +113,7 @@ impl<'tcx> QueryLatch<'tcx> {
112113
// we have to be in the `wait` call. This is ensured by the deadlock handler
113114
// getting the self.info lock.
114115
rustc_thread_pool::mark_blocked();
116+
collect::release();
115117
tcx.jobserver_proxy.release_thread();
116118
waiter.condvar.wait(&mut waiters_guard);
117119
// Release the lock before we potentially block in `acquire_thread`

compiler/rustc_middle/src/query/keys.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ use crate::{mir, traits};
2424
#[derive(Copy, Clone, Debug)]
2525
pub struct LocalCrate;
2626

27-
pub trait QueryKeyBounds = Copy + Debug + Eq + Hash + for<'a> HashStable<StableHashingContext<'a>>;
27+
pub trait QueryKeyBounds =
28+
Copy + Debug + Eq + Hash + Send + for<'a> HashStable<StableHashingContext<'a>>;
2829

2930
/// Controls what types can legally be used as the key for a query.
3031
pub trait QueryKey: Sized + QueryKeyBounds {

compiler/rustc_query_impl/src/execution.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::mem::ManuallyDrop;
33

44
use rustc_data_structures::hash_table::{Entry, HashTable};
55
use rustc_data_structures::stack::ensure_sufficient_stack;
6-
use rustc_data_structures::sync::{DynSend, DynSync};
6+
use rustc_data_structures::sync::{DynSend, DynSync, collect};
77
use rustc_data_structures::{outline, sharded, sync};
88
use rustc_errors::FatalError;
99
use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
@@ -331,6 +331,10 @@ fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
331331
// `query.state`.
332332
job_guard.complete(&query.cache, value, dep_node_index);
333333

334+
// Periodic memory reclamation trigger.
335+
// We do this after `complete` since the default caches may have memory to free due to table expansion.
336+
collect::collect();
337+
334338
(value, Some(dep_node_index))
335339
}
336340
Entry::Occupied(mut entry) => {

0 commit comments

Comments
 (0)