1 //! The implementation of the query system itself. This defines the macros that
2 //! generate the actual methods on tcx which find and execute the provider,
3 //! manage the caches, and so forth.
4
5 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex, DepNodeParams};
6 use crate::query::caches::QueryCache;
7 use crate::query::config::{QueryDescription, QueryVtable};
8 use crate::query::job::{
9 report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId,
10 };
11 use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
12
13 use rustc_data_structures::fingerprint::Fingerprint;
14 use rustc_data_structures::fx::{FxHashMap, FxHasher};
15 #[cfg(parallel_compiler)]
16 use rustc_data_structures::profiling::TimingGuard;
17 use rustc_data_structures::sharded::{get_shard_index_by_hash, Sharded};
18 use rustc_data_structures::sync::{Lock, LockGuard};
19 use rustc_data_structures::thin_vec::ThinVec;
20 use rustc_errors::{DiagnosticBuilder, FatalError};
21 use rustc_session::Session;
22 use rustc_span::{Span, DUMMY_SP};
23 use std::cell::Cell;
24 use std::collections::hash_map::Entry;
25 use std::fmt::Debug;
26 use std::hash::{Hash, Hasher};
27 use std::mem;
28 use std::num::NonZeroU32;
29 use std::ptr;
30
31 pub struct QueryCacheStore<C: QueryCache> {
32 cache: C,
33 shards: Sharded<C::Sharded>,
34 }
35
36 impl<C: QueryCache + Default> Default for QueryCacheStore<C> {
default() -> Self37 fn default() -> Self {
38 Self { cache: C::default(), shards: Default::default() }
39 }
40 }
41
42 /// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
43 pub struct QueryLookup {
44 pub(super) key_hash: u64,
45 shard: usize,
46 }
47
48 // We compute the key's hash once and then use it for both the
49 // shard lookup and the hashmap lookup. This relies on the fact
50 // that both of them use `FxHasher`.
hash_for_shard<K: Hash>(key: &K) -> u6451 fn hash_for_shard<K: Hash>(key: &K) -> u64 {
52 let mut hasher = FxHasher::default();
53 key.hash(&mut hasher);
54 hasher.finish()
55 }
56
57 impl<C: QueryCache> QueryCacheStore<C> {
get_lookup<'tcx>( &'tcx self, key: &C::Key, ) -> (QueryLookup, LockGuard<'tcx, C::Sharded>)58 pub(super) fn get_lookup<'tcx>(
59 &'tcx self,
60 key: &C::Key,
61 ) -> (QueryLookup, LockGuard<'tcx, C::Sharded>) {
62 let key_hash = hash_for_shard(key);
63 let shard = get_shard_index_by_hash(key_hash);
64 let lock = self.shards.get_shard_by_index(shard).lock();
65 (QueryLookup { key_hash, shard }, lock)
66 }
67
iter_results(&self, f: &mut dyn FnMut(&C::Key, &C::Value, DepNodeIndex))68 pub fn iter_results(&self, f: &mut dyn FnMut(&C::Key, &C::Value, DepNodeIndex)) {
69 self.cache.iter(&self.shards, f)
70 }
71 }
72
73 struct QueryStateShard<D, K> {
74 active: FxHashMap<K, QueryResult<D>>,
75
76 /// Used to generate unique ids for active jobs.
77 jobs: u32,
78 }
79
80 impl<D, K> Default for QueryStateShard<D, K> {
default() -> QueryStateShard<D, K>81 fn default() -> QueryStateShard<D, K> {
82 QueryStateShard { active: Default::default(), jobs: 0 }
83 }
84 }
85
86 pub struct QueryState<D, K> {
87 shards: Sharded<QueryStateShard<D, K>>,
88 }
89
90 /// Indicates the state of a query for a given key in a query map.
91 enum QueryResult<D> {
92 /// An already executing query. The query job can be used to await for its completion.
93 Started(QueryJob<D>),
94
95 /// The query panicked. Queries trying to wait on this will raise a fatal error which will
96 /// silently panic.
97 Poisoned,
98 }
99
100 impl<D, K> QueryState<D, K>
101 where
102 D: Copy + Clone + Eq + Hash,
103 K: Eq + Hash + Clone + Debug,
104 {
all_inactive(&self) -> bool105 pub fn all_inactive(&self) -> bool {
106 let shards = self.shards.lock_shards();
107 shards.iter().all(|shard| shard.active.is_empty())
108 }
109
try_collect_active_jobs<CTX: Copy>( &self, tcx: CTX, kind: D, make_query: fn(CTX, K) -> QueryStackFrame, jobs: &mut QueryMap<D>, ) -> Option<()>110 pub fn try_collect_active_jobs<CTX: Copy>(
111 &self,
112 tcx: CTX,
113 kind: D,
114 make_query: fn(CTX, K) -> QueryStackFrame,
115 jobs: &mut QueryMap<D>,
116 ) -> Option<()> {
117 // We use try_lock_shards here since we are called from the
118 // deadlock handler, and this shouldn't be locked.
119 let shards = self.shards.try_lock_shards()?;
120 for (shard_id, shard) in shards.iter().enumerate() {
121 for (k, v) in shard.active.iter() {
122 if let QueryResult::Started(ref job) = *v {
123 let id = QueryJobId::new(job.id, shard_id, kind);
124 let query = make_query(tcx, k.clone());
125 jobs.insert(id, QueryJobInfo { query, job: job.clone() });
126 }
127 }
128 }
129
130 Some(())
131 }
132 }
133
134 impl<D, K> Default for QueryState<D, K> {
default() -> QueryState<D, K>135 fn default() -> QueryState<D, K> {
136 QueryState { shards: Default::default() }
137 }
138 }
139
140 /// A type representing the responsibility to execute the job in the `job` field.
141 /// This will poison the relevant query if dropped.
142 struct JobOwner<'tcx, D, K>
143 where
144 D: Copy + Clone + Eq + Hash,
145 K: Eq + Hash + Clone,
146 {
147 state: &'tcx QueryState<D, K>,
148 key: K,
149 id: QueryJobId<D>,
150 }
151
152 #[cold]
153 #[inline(never)]
mk_cycle<CTX, V, R>( tcx: CTX, error: CycleError, handle_cycle_error: fn(CTX, DiagnosticBuilder<'_>) -> V, cache: &dyn crate::query::QueryStorage<Value = V, Stored = R>, ) -> R where CTX: QueryContext, V: std::fmt::Debug, R: Clone,154 fn mk_cycle<CTX, V, R>(
155 tcx: CTX,
156 error: CycleError,
157 handle_cycle_error: fn(CTX, DiagnosticBuilder<'_>) -> V,
158 cache: &dyn crate::query::QueryStorage<Value = V, Stored = R>,
159 ) -> R
160 where
161 CTX: QueryContext,
162 V: std::fmt::Debug,
163 R: Clone,
164 {
165 let error = report_cycle(tcx.dep_context().sess(), error);
166 let value = handle_cycle_error(tcx, error);
167 cache.store_nocache(value)
168 }
169
170 impl<'tcx, D, K> JobOwner<'tcx, D, K>
171 where
172 D: Copy + Clone + Eq + Hash,
173 K: Eq + Hash + Clone,
174 {
175 /// Either gets a `JobOwner` corresponding the query, allowing us to
176 /// start executing the query, or returns with the result of the query.
177 /// This function assumes that `try_get_cached` is already called and returned `lookup`.
178 /// If the query is executing elsewhere, this will wait for it and return the result.
179 /// If the query panicked, this will silently panic.
180 ///
181 /// This function is inlined because that results in a noticeable speed-up
182 /// for some compile-time benchmarks.
183 #[inline(always)]
try_start<'b, CTX>( tcx: &'b CTX, state: &'b QueryState<CTX::DepKind, K>, span: Span, key: K, lookup: QueryLookup, dep_kind: CTX::DepKind, ) -> TryGetJob<'b, CTX::DepKind, K> where CTX: QueryContext,184 fn try_start<'b, CTX>(
185 tcx: &'b CTX,
186 state: &'b QueryState<CTX::DepKind, K>,
187 span: Span,
188 key: K,
189 lookup: QueryLookup,
190 dep_kind: CTX::DepKind,
191 ) -> TryGetJob<'b, CTX::DepKind, K>
192 where
193 CTX: QueryContext,
194 {
195 let shard = lookup.shard;
196 let mut state_lock = state.shards.get_shard_by_index(shard).lock();
197 let lock = &mut *state_lock;
198
199 match lock.active.entry(key) {
200 Entry::Vacant(entry) => {
201 // Generate an id unique within this shard.
202 let id = lock.jobs.checked_add(1).unwrap();
203 lock.jobs = id;
204 let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
205
206 let job = tcx.current_query_job();
207 let job = QueryJob::new(id, span, job);
208
209 let key = entry.key().clone();
210 entry.insert(QueryResult::Started(job));
211
212 let global_id = QueryJobId::new(id, shard, dep_kind);
213 let owner = JobOwner { state, id: global_id, key };
214 return TryGetJob::NotYetStarted(owner);
215 }
216 Entry::Occupied(mut entry) => {
217 match entry.get_mut() {
218 #[cfg(not(parallel_compiler))]
219 QueryResult::Started(job) => {
220 let id = QueryJobId::new(job.id, shard, dep_kind);
221
222 drop(state_lock);
223
224 // If we are single-threaded we know that we have cycle error,
225 // so we just return the error.
226 return TryGetJob::Cycle(id.find_cycle_in_stack(
227 tcx.try_collect_active_jobs().unwrap(),
228 &tcx.current_query_job(),
229 span,
230 ));
231 }
232 #[cfg(parallel_compiler)]
233 QueryResult::Started(job) => {
234 // For parallel queries, we'll block and wait until the query running
235 // in another thread has completed. Record how long we wait in the
236 // self-profiler.
237 let query_blocked_prof_timer = tcx.dep_context().profiler().query_blocked();
238
239 // Get the latch out
240 let latch = job.latch();
241
242 drop(state_lock);
243
244 // With parallel queries we might just have to wait on some other
245 // thread.
246 let result = latch.wait_on(tcx.current_query_job(), span);
247
248 match result {
249 Ok(()) => TryGetJob::JobCompleted(query_blocked_prof_timer),
250 Err(cycle) => TryGetJob::Cycle(cycle),
251 }
252 }
253 QueryResult::Poisoned => FatalError.raise(),
254 }
255 }
256 }
257 }
258
259 /// Completes the query by updating the query cache with the `result`,
260 /// signals the waiter and forgets the JobOwner, so it won't poison the query
complete<C>( self, cache: &QueryCacheStore<C>, result: C::Value, dep_node_index: DepNodeIndex, ) -> C::Stored where C: QueryCache<Key = K>,261 fn complete<C>(
262 self,
263 cache: &QueryCacheStore<C>,
264 result: C::Value,
265 dep_node_index: DepNodeIndex,
266 ) -> C::Stored
267 where
268 C: QueryCache<Key = K>,
269 {
270 // We can move out of `self` here because we `mem::forget` it below
271 let key = unsafe { ptr::read(&self.key) };
272 let state = self.state;
273
274 // Forget ourself so our destructor won't poison the query
275 mem::forget(self);
276
277 let (job, result) = {
278 let key_hash = hash_for_shard(&key);
279 let shard = get_shard_index_by_hash(key_hash);
280 let job = {
281 let mut lock = state.shards.get_shard_by_index(shard).lock();
282 match lock.active.remove(&key).unwrap() {
283 QueryResult::Started(job) => job,
284 QueryResult::Poisoned => panic!(),
285 }
286 };
287 let result = {
288 let mut lock = cache.shards.get_shard_by_index(shard).lock();
289 cache.cache.complete(&mut lock, key, result, dep_node_index)
290 };
291 (job, result)
292 };
293
294 job.signal_complete();
295 result
296 }
297 }
298
299 impl<'tcx, D, K> Drop for JobOwner<'tcx, D, K>
300 where
301 D: Copy + Clone + Eq + Hash,
302 K: Eq + Hash + Clone,
303 {
304 #[inline(never)]
305 #[cold]
drop(&mut self)306 fn drop(&mut self) {
307 // Poison the query so jobs waiting on it panic.
308 let state = self.state;
309 let shard = state.shards.get_shard_by_value(&self.key);
310 let job = {
311 let mut shard = shard.lock();
312 let job = match shard.active.remove(&self.key).unwrap() {
313 QueryResult::Started(job) => job,
314 QueryResult::Poisoned => panic!(),
315 };
316 shard.active.insert(self.key.clone(), QueryResult::Poisoned);
317 job
318 };
319 // Also signal the completion of the job, so waiters
320 // will continue execution.
321 job.signal_complete();
322 }
323 }
324
325 #[derive(Clone)]
326 pub(crate) struct CycleError {
327 /// The query and related span that uses the cycle.
328 pub usage: Option<(Span, QueryStackFrame)>,
329 pub cycle: Vec<QueryInfo>,
330 }
331
332 /// The result of `try_start`.
333 enum TryGetJob<'tcx, D, K>
334 where
335 D: Copy + Clone + Eq + Hash,
336 K: Eq + Hash + Clone,
337 {
338 /// The query is not yet started. Contains a guard to the cache eventually used to start it.
339 NotYetStarted(JobOwner<'tcx, D, K>),
340
341 /// The query was already completed.
342 /// Returns the result of the query and its dep-node index
343 /// if it succeeded or a cycle error if it failed.
344 #[cfg(parallel_compiler)]
345 JobCompleted(TimingGuard<'tcx>),
346
347 /// Trying to execute the query resulted in a cycle.
348 Cycle(CycleError),
349 }
350
351 /// Checks if the query is already computed and in the cache.
352 /// It returns the shard index and a lock guard to the shard,
353 /// which will be used if the query is not in the cache and we need
354 /// to compute it.
355 #[inline]
try_get_cached<'a, CTX, C, R, OnHit>( tcx: CTX, cache: &'a QueryCacheStore<C>, key: &C::Key, on_hit: OnHit, ) -> Result<R, QueryLookup> where C: QueryCache, CTX: DepContext, OnHit: FnOnce(&C::Stored) -> R,356 pub fn try_get_cached<'a, CTX, C, R, OnHit>(
357 tcx: CTX,
358 cache: &'a QueryCacheStore<C>,
359 key: &C::Key,
360 // `on_hit` can be called while holding a lock to the query cache
361 on_hit: OnHit,
362 ) -> Result<R, QueryLookup>
363 where
364 C: QueryCache,
365 CTX: DepContext,
366 OnHit: FnOnce(&C::Stored) -> R,
367 {
368 cache.cache.lookup(cache, &key, |value, index| {
369 if unlikely!(tcx.profiler().enabled()) {
370 tcx.profiler().query_cache_hit(index.into());
371 }
372 tcx.dep_graph().read_index(index);
373 on_hit(value)
374 })
375 }
376
try_execute_query<CTX, C>( tcx: CTX, state: &QueryState<CTX::DepKind, C::Key>, cache: &QueryCacheStore<C>, span: Span, key: C::Key, lookup: QueryLookup, dep_node: Option<DepNode<CTX::DepKind>>, query: &QueryVtable<CTX, C::Key, C::Value>, ) -> (C::Stored, Option<DepNodeIndex>) where C: QueryCache, C::Key: Clone + DepNodeParams<CTX::DepContext>, CTX: QueryContext,377 fn try_execute_query<CTX, C>(
378 tcx: CTX,
379 state: &QueryState<CTX::DepKind, C::Key>,
380 cache: &QueryCacheStore<C>,
381 span: Span,
382 key: C::Key,
383 lookup: QueryLookup,
384 dep_node: Option<DepNode<CTX::DepKind>>,
385 query: &QueryVtable<CTX, C::Key, C::Value>,
386 ) -> (C::Stored, Option<DepNodeIndex>)
387 where
388 C: QueryCache,
389 C::Key: Clone + DepNodeParams<CTX::DepContext>,
390 CTX: QueryContext,
391 {
392 match JobOwner::<'_, CTX::DepKind, C::Key>::try_start(
393 &tcx,
394 state,
395 span,
396 key.clone(),
397 lookup,
398 query.dep_kind,
399 ) {
400 TryGetJob::NotYetStarted(job) => {
401 let (result, dep_node_index) = execute_job(tcx, key, dep_node, query, job.id);
402 let result = job.complete(cache, result, dep_node_index);
403 (result, Some(dep_node_index))
404 }
405 TryGetJob::Cycle(error) => {
406 let result = mk_cycle(tcx, error, query.handle_cycle_error, &cache.cache);
407 (result, None)
408 }
409 #[cfg(parallel_compiler)]
410 TryGetJob::JobCompleted(query_blocked_prof_timer) => {
411 let (v, index) = cache
412 .cache
413 .lookup(cache, &key, |value, index| (value.clone(), index))
414 .unwrap_or_else(|_| panic!("value must be in cache after waiting"));
415
416 if unlikely!(tcx.dep_context().profiler().enabled()) {
417 tcx.dep_context().profiler().query_cache_hit(index.into());
418 }
419 query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
420
421 (v, Some(index))
422 }
423 }
424 }
425
execute_job<CTX, K, V>( tcx: CTX, key: K, mut dep_node_opt: Option<DepNode<CTX::DepKind>>, query: &QueryVtable<CTX, K, V>, job_id: QueryJobId<CTX::DepKind>, ) -> (V, DepNodeIndex) where K: Clone + DepNodeParams<CTX::DepContext>, V: Debug, CTX: QueryContext,426 fn execute_job<CTX, K, V>(
427 tcx: CTX,
428 key: K,
429 mut dep_node_opt: Option<DepNode<CTX::DepKind>>,
430 query: &QueryVtable<CTX, K, V>,
431 job_id: QueryJobId<CTX::DepKind>,
432 ) -> (V, DepNodeIndex)
433 where
434 K: Clone + DepNodeParams<CTX::DepContext>,
435 V: Debug,
436 CTX: QueryContext,
437 {
438 let dep_graph = tcx.dep_context().dep_graph();
439
440 // Fast path for when incr. comp. is off.
441 if !dep_graph.is_fully_enabled() {
442 let prof_timer = tcx.dep_context().profiler().query_provider();
443 let result = tcx.start_query(job_id, None, || query.compute(*tcx.dep_context(), key));
444 let dep_node_index = dep_graph.next_virtual_depnode_index();
445 prof_timer.finish_with_query_invocation_id(dep_node_index.into());
446 return (result, dep_node_index);
447 }
448
449 if !query.anon && !query.eval_always {
450 // `to_dep_node` is expensive for some `DepKind`s.
451 let dep_node =
452 dep_node_opt.get_or_insert_with(|| query.to_dep_node(*tcx.dep_context(), &key));
453
454 // The diagnostics for this query will be promoted to the current session during
455 // `try_mark_green()`, so we can ignore them here.
456 if let Some(ret) = tcx.start_query(job_id, None, || {
457 try_load_from_disk_and_cache_in_memory(tcx, &key, &dep_node, query)
458 }) {
459 return ret;
460 }
461 }
462
463 let prof_timer = tcx.dep_context().profiler().query_provider();
464 let diagnostics = Lock::new(ThinVec::new());
465
466 let (result, dep_node_index) = tcx.start_query(job_id, Some(&diagnostics), || {
467 if query.anon {
468 return dep_graph.with_anon_task(*tcx.dep_context(), query.dep_kind, || {
469 query.compute(*tcx.dep_context(), key)
470 });
471 }
472
473 // `to_dep_node` is expensive for some `DepKind`s.
474 let dep_node = dep_node_opt.unwrap_or_else(|| query.to_dep_node(*tcx.dep_context(), &key));
475
476 dep_graph.with_task(dep_node, *tcx.dep_context(), key, query.compute, query.hash_result)
477 });
478
479 prof_timer.finish_with_query_invocation_id(dep_node_index.into());
480
481 let diagnostics = diagnostics.into_inner();
482 let side_effects = QuerySideEffects { diagnostics };
483
484 if unlikely!(!side_effects.is_empty()) {
485 if query.anon {
486 tcx.store_side_effects_for_anon_node(dep_node_index, side_effects);
487 } else {
488 tcx.store_side_effects(dep_node_index, side_effects);
489 }
490 }
491
492 (result, dep_node_index)
493 }
494
try_load_from_disk_and_cache_in_memory<CTX, K, V>( tcx: CTX, key: &K, dep_node: &DepNode<CTX::DepKind>, query: &QueryVtable<CTX, K, V>, ) -> Option<(V, DepNodeIndex)> where K: Clone, CTX: QueryContext, V: Debug,495 fn try_load_from_disk_and_cache_in_memory<CTX, K, V>(
496 tcx: CTX,
497 key: &K,
498 dep_node: &DepNode<CTX::DepKind>,
499 query: &QueryVtable<CTX, K, V>,
500 ) -> Option<(V, DepNodeIndex)>
501 where
502 K: Clone,
503 CTX: QueryContext,
504 V: Debug,
505 {
506 // Note this function can be called concurrently from the same query
507 // We must ensure that this is handled correctly.
508
509 let dep_graph = tcx.dep_context().dep_graph();
510 let (prev_dep_node_index, dep_node_index) = dep_graph.try_mark_green(tcx, &dep_node)?;
511
512 debug_assert!(dep_graph.is_green(dep_node));
513
514 // First we try to load the result from the on-disk cache.
515 // Some things are never cached on disk.
516 if query.cache_on_disk {
517 let prof_timer = tcx.dep_context().profiler().incr_cache_loading();
518 let result = query.try_load_from_disk(tcx, prev_dep_node_index);
519 prof_timer.finish_with_query_invocation_id(dep_node_index.into());
520
521 if let Some(result) = result {
522 let prev_fingerprint = tcx
523 .dep_context()
524 .dep_graph()
525 .prev_fingerprint_of(dep_node)
526 .unwrap_or(Fingerprint::ZERO);
527 // If `-Zincremental-verify-ich` is specified, re-hash results from
528 // the cache and make sure that they have the expected fingerprint.
529 //
530 // If not, we still seek to verify a subset of fingerprints loaded
531 // from disk. Re-hashing results is fairly expensive, so we can't
532 // currently afford to verify every hash. This subset should still
533 // give us some coverage of potential bugs though.
534 let try_verify = prev_fingerprint.as_value().1 % 32 == 0;
535 if unlikely!(
536 try_verify || tcx.dep_context().sess().opts.debugging_opts.incremental_verify_ich
537 ) {
538 incremental_verify_ich(*tcx.dep_context(), &result, dep_node, query);
539 }
540
541 return Some((result, dep_node_index));
542 }
543
544 // We always expect to find a cached result for things that
545 // can be forced from `DepNode`.
546 debug_assert!(
547 !tcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
548 "missing on-disk cache entry for {:?}",
549 dep_node
550 );
551 }
552
553 // We could not load a result from the on-disk cache, so
554 // recompute.
555 let prof_timer = tcx.dep_context().profiler().query_provider();
556
557 // The dep-graph for this computation is already in-place.
558 let result = dep_graph.with_ignore(|| query.compute(*tcx.dep_context(), key.clone()));
559
560 prof_timer.finish_with_query_invocation_id(dep_node_index.into());
561
562 // Verify that re-running the query produced a result with the expected hash
563 // This catches bugs in query implementations, turning them into ICEs.
564 // For example, a query might sort its result by `DefId` - since `DefId`s are
565 // not stable across compilation sessions, the result could get up getting sorted
566 // in a different order when the query is re-run, even though all of the inputs
567 // (e.g. `DefPathHash` values) were green.
568 //
569 // See issue #82920 for an example of a miscompilation that would get turned into
570 // an ICE by this check
571 incremental_verify_ich(*tcx.dep_context(), &result, dep_node, query);
572
573 Some((result, dep_node_index))
574 }
575
incremental_verify_ich<CTX, K, V: Debug>( tcx: CTX::DepContext, result: &V, dep_node: &DepNode<CTX::DepKind>, query: &QueryVtable<CTX, K, V>, ) where CTX: QueryContext,576 fn incremental_verify_ich<CTX, K, V: Debug>(
577 tcx: CTX::DepContext,
578 result: &V,
579 dep_node: &DepNode<CTX::DepKind>,
580 query: &QueryVtable<CTX, K, V>,
581 ) where
582 CTX: QueryContext,
583 {
584 assert!(
585 tcx.dep_graph().is_green(dep_node),
586 "fingerprint for green query instance not loaded from cache: {:?}",
587 dep_node,
588 );
589
590 debug!("BEGIN verify_ich({:?})", dep_node);
591 let new_hash = query.hash_result.map_or(Fingerprint::ZERO, |f| {
592 let mut hcx = tcx.create_stable_hashing_context();
593 f(&mut hcx, result)
594 });
595 let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node);
596 debug!("END verify_ich({:?})", dep_node);
597
598 if Some(new_hash) != old_hash {
599 incremental_verify_ich_cold(tcx.sess(), DebugArg::from(&dep_node), DebugArg::from(&result));
600 }
601 }
602
603 // This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
604 // currently not exposed publicly.
605 //
606 // The PR which added this attempted to use `&dyn Debug` instead, but that
607 // showed statistically significant worse compiler performance. It's not
608 // actually clear what the cause there was -- the code should be cold. If this
609 // can be replaced with `&dyn Debug` with on perf impact, then it probably
610 // should be.
611 extern "C" {
612 type Opaque;
613 }
614
615 struct DebugArg<'a> {
616 value: &'a Opaque,
617 fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result,
618 }
619
620 impl<'a, T> From<&'a T> for DebugArg<'a>
621 where
622 T: std::fmt::Debug,
623 {
from(value: &'a T) -> DebugArg<'a>624 fn from(value: &'a T) -> DebugArg<'a> {
625 DebugArg {
626 value: unsafe { std::mem::transmute(value) },
627 fmt: unsafe {
628 std::mem::transmute(<T as std::fmt::Debug>::fmt as fn(_, _) -> std::fmt::Result)
629 },
630 }
631 }
632 }
633
634 impl std::fmt::Debug for DebugArg<'_> {
fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result635 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
636 (self.fmt)(self.value, f)
637 }
638 }
639
640 // Note that this is marked #[cold] and intentionally takes the equivalent of
641 // `dyn Debug` for its arguments, as we want to avoid generating a bunch of
642 // different implementations for LLVM to chew on (and filling up the final
643 // binary, too).
644 #[cold]
incremental_verify_ich_cold(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>)645 fn incremental_verify_ich_cold(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) {
646 let run_cmd = if let Some(crate_name) = &sess.opts.crate_name {
647 format!("`cargo clean -p {}` or `cargo clean`", crate_name)
648 } else {
649 "`cargo clean`".to_string()
650 };
651
652 // When we emit an error message and panic, we try to debug-print the `DepNode`
653 // and query result. Unfortunately, this can cause us to run additional queries,
654 // which may result in another fingerprint mismatch while we're in the middle
655 // of processing this one. To avoid a double-panic (which kills the process
656 // before we can print out the query static), we print out a terse
657 // but 'safe' message if we detect a re-entrant call to this method.
658 thread_local! {
659 static INSIDE_VERIFY_PANIC: Cell<bool> = const { Cell::new(false) };
660 };
661
662 let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
663
664 if old_in_panic {
665 sess.struct_err(
666 "internal compiler error: re-entrant incremental verify failure, suppressing message",
667 )
668 .emit();
669 } else {
670 sess.struct_err(&format!("internal compiler error: encountered incremental compilation error with {:?}", dep_node))
671 .help(&format!("This is a known issue with the compiler. Run {} to allow your project to compile", run_cmd))
672 .note(&"Please follow the instructions below to create a bug report with the provided information")
673 .note(&"See <https://github.com/rust-lang/rust/issues/84970> for more information")
674 .emit();
675 panic!("Found unstable fingerprints for {:?}: {:?}", dep_node, result);
676 }
677
678 INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));
679 }
680
681 /// Ensure that either this query has all green inputs or been executed.
682 /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
683 /// Returns true if the query should still run.
684 ///
685 /// This function is particularly useful when executing passes for their
686 /// side-effects -- e.g., in order to report errors for erroneous programs.
687 ///
688 /// Note: The optimization is only available during incr. comp.
689 #[inline(never)]
ensure_must_run<CTX, K, V>( tcx: CTX, key: &K, query: &QueryVtable<CTX, K, V>, ) -> (bool, Option<DepNode<CTX::DepKind>>) where K: crate::dep_graph::DepNodeParams<CTX::DepContext>, CTX: QueryContext,690 fn ensure_must_run<CTX, K, V>(
691 tcx: CTX,
692 key: &K,
693 query: &QueryVtable<CTX, K, V>,
694 ) -> (bool, Option<DepNode<CTX::DepKind>>)
695 where
696 K: crate::dep_graph::DepNodeParams<CTX::DepContext>,
697 CTX: QueryContext,
698 {
699 if query.eval_always {
700 return (true, None);
701 }
702
703 // Ensuring an anonymous query makes no sense
704 assert!(!query.anon);
705
706 let dep_node = query.to_dep_node(*tcx.dep_context(), key);
707
708 let dep_graph = tcx.dep_context().dep_graph();
709 match dep_graph.try_mark_green(tcx, &dep_node) {
710 None => {
711 // A None return from `try_mark_green` means that this is either
712 // a new dep node or that the dep node has already been marked red.
713 // Either way, we can't call `dep_graph.read()` as we don't have the
714 // DepNodeIndex. We must invoke the query itself. The performance cost
715 // this introduces should be negligible as we'll immediately hit the
716 // in-memory cache, or another query down the line will.
717 (true, Some(dep_node))
718 }
719 Some((_, dep_node_index)) => {
720 dep_graph.read_index(dep_node_index);
721 tcx.dep_context().profiler().query_cache_hit(dep_node_index.into());
722 (false, None)
723 }
724 }
725 }
726
727 pub enum QueryMode {
728 Get,
729 Ensure,
730 }
731
get_query<Q, CTX>( tcx: CTX, span: Span, key: Q::Key, lookup: QueryLookup, mode: QueryMode, ) -> Option<Q::Stored> where Q: QueryDescription<CTX>, Q::Key: DepNodeParams<CTX::DepContext>, CTX: QueryContext,732 pub fn get_query<Q, CTX>(
733 tcx: CTX,
734 span: Span,
735 key: Q::Key,
736 lookup: QueryLookup,
737 mode: QueryMode,
738 ) -> Option<Q::Stored>
739 where
740 Q: QueryDescription<CTX>,
741 Q::Key: DepNodeParams<CTX::DepContext>,
742 CTX: QueryContext,
743 {
744 let query = Q::make_vtable(tcx, &key);
745 let dep_node = if let QueryMode::Ensure = mode {
746 let (must_run, dep_node) = ensure_must_run(tcx, &key, &query);
747 if !must_run {
748 return None;
749 }
750 dep_node
751 } else {
752 None
753 };
754
755 debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
756 let (result, dep_node_index) = try_execute_query(
757 tcx,
758 Q::query_state(tcx),
759 Q::query_cache(tcx),
760 span,
761 key,
762 lookup,
763 dep_node,
764 &query,
765 );
766 if let Some(dep_node_index) = dep_node_index {
767 tcx.dep_context().dep_graph().read_index(dep_node_index)
768 }
769 Some(result)
770 }
771
force_query<Q, CTX>(tcx: CTX, key: Q::Key, dep_node: DepNode<CTX::DepKind>) where Q: QueryDescription<CTX>, Q::Key: DepNodeParams<CTX::DepContext>, CTX: QueryContext,772 pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, dep_node: DepNode<CTX::DepKind>)
773 where
774 Q: QueryDescription<CTX>,
775 Q::Key: DepNodeParams<CTX::DepContext>,
776 CTX: QueryContext,
777 {
778 // We may be concurrently trying both execute and force a query.
779 // Ensure that only one of them runs the query.
780 let cache = Q::query_cache(tcx);
781 let cached = cache.cache.lookup(cache, &key, |_, index| {
782 if unlikely!(tcx.dep_context().profiler().enabled()) {
783 tcx.dep_context().profiler().query_cache_hit(index.into());
784 }
785 });
786
787 let lookup = match cached {
788 Ok(()) => return,
789 Err(lookup) => lookup,
790 };
791
792 let query = Q::make_vtable(tcx, &key);
793 let state = Q::query_state(tcx);
794 debug_assert!(!query.anon);
795
796 try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), &query);
797 }
798