1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * IRQ subsystem internal functions and variables: 4dbec07baSThomas Gleixner * 5dbec07baSThomas Gleixner * Do not ever include this file from anything else than 6dbec07baSThomas Gleixner * kernel/irq/. Do not even think about using any information outside 7dbec07baSThomas Gleixner * of this file for your non core code. 81da177e4SLinus Torvalds */ 9e144710bSThomas Gleixner #include <linux/irqdesc.h> 108f945a33SThomas Gleixner #include <linux/kernel_stat.h> 11be45beb2SJon Hunter #include <linux/pm_runtime.h> 12b2d3d61aSDaniel Lezcano #include <linux/sched/clock.h> 131da177e4SLinus Torvalds 14c1ee6264SThomas Gleixner #ifdef CONFIG_SPARSE_IRQ 15c1ee6264SThomas Gleixner # define IRQ_BITMAP_BITS (NR_IRQS + 8196) 16c1ee6264SThomas Gleixner #else 17c1ee6264SThomas Gleixner # define IRQ_BITMAP_BITS NR_IRQS 18c1ee6264SThomas Gleixner #endif 19c1ee6264SThomas Gleixner 20dbec07baSThomas Gleixner #define istate core_internal_state__do_not_mess_with_it 21dbec07baSThomas Gleixner 222329abfaSRusty Russell extern bool noirqdebug; 231da177e4SLinus Torvalds 24e509bd7dSMika Westerberg extern struct irqaction chained_action; 25e509bd7dSMika Westerberg 261535dfacSThomas Gleixner /* 271535dfacSThomas Gleixner * Bits used by threaded handlers: 281535dfacSThomas Gleixner * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run 291535dfacSThomas Gleixner * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed 301535dfacSThomas Gleixner * IRQTF_AFFINITY - irq thread is requested to adjust affinity 318d32a307SThomas Gleixner * IRQTF_FORCED_THREAD - irq action is force threaded 321535dfacSThomas Gleixner */ 331535dfacSThomas Gleixner enum { 341535dfacSThomas Gleixner IRQTF_RUNTHREAD, 351535dfacSThomas Gleixner IRQTF_WARNED, 361535dfacSThomas Gleixner IRQTF_AFFINITY, 378d32a307SThomas Gleixner IRQTF_FORCED_THREAD, 381535dfacSThomas Gleixner }; 391535dfacSThomas Gleixner 40bd062e76SThomas Gleixner /* 41a257954bSJiang Liu * Bit masks for desc->core_internal_state__do_not_mess_with_it 42bd062e76SThomas Gleixner * 43bd062e76SThomas Gleixner * IRQS_AUTODETECT - autodetection in progress 447acdd53eSThomas Gleixner * IRQS_SPURIOUS_DISABLED - was disabled due to spurious interrupt 457acdd53eSThomas Gleixner * detection 466954b75bSThomas Gleixner * IRQS_POLL_INPROGRESS - polling in progress 473d67baecSThomas Gleixner * IRQS_ONESHOT - irq is not unmasked in primary handler 48163ef309SThomas Gleixner * IRQS_REPLAY - irq is replayed 49163ef309SThomas Gleixner * IRQS_WAITING - irq is waiting 502a0d6fb3SThomas Gleixner * IRQS_PENDING - irq is pending and replayed later 51c531e836SThomas Gleixner * IRQS_SUSPENDED - irq is suspended 52bd062e76SThomas Gleixner */ 53bd062e76SThomas Gleixner enum { 54bd062e76SThomas Gleixner IRQS_AUTODETECT = 0x00000001, 557acdd53eSThomas Gleixner IRQS_SPURIOUS_DISABLED = 0x00000002, 566954b75bSThomas Gleixner IRQS_POLL_INPROGRESS = 0x00000008, 573d67baecSThomas Gleixner IRQS_ONESHOT = 0x00000020, 58163ef309SThomas Gleixner IRQS_REPLAY = 0x00000040, 59163ef309SThomas Gleixner IRQS_WAITING = 0x00000080, 602a0d6fb3SThomas Gleixner IRQS_PENDING = 0x00000200, 61c531e836SThomas Gleixner IRQS_SUSPENDED = 0x00000800, 62b2d3d61aSDaniel Lezcano IRQS_TIMINGS = 0x00001000, 63bd062e76SThomas Gleixner }; 64bd062e76SThomas Gleixner 651ce6068dSThomas Gleixner #include "debug.h" 661ce6068dSThomas Gleixner #include "settings.h" 671ce6068dSThomas Gleixner 68a1ff541aSJiang Liu extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags); 6979ff1cdaSJiang Liu extern void __disable_irq(struct irq_desc *desc); 7079ff1cdaSJiang Liu extern void __enable_irq(struct irq_desc *desc); 710c5d1eb7SDavid Brownell 724cde9c6bSThomas Gleixner #define IRQ_RESEND true 734cde9c6bSThomas Gleixner #define IRQ_NORESEND false 744cde9c6bSThomas Gleixner 754cde9c6bSThomas Gleixner #define IRQ_START_FORCE true 764cde9c6bSThomas Gleixner #define IRQ_START_COND false 774cde9c6bSThomas Gleixner 78c942cee4SThomas Gleixner extern int irq_activate(struct irq_desc *desc); 791beaeacdSThomas Gleixner extern int irq_activate_and_startup(struct irq_desc *desc, bool resend); 804cde9c6bSThomas Gleixner extern int irq_startup(struct irq_desc *desc, bool resend, bool force); 814cde9c6bSThomas Gleixner 8246999238SThomas Gleixner extern void irq_shutdown(struct irq_desc *desc); 8387923470SThomas Gleixner extern void irq_enable(struct irq_desc *desc); 8487923470SThomas Gleixner extern void irq_disable(struct irq_desc *desc); 8531d9d9b6SMarc Zyngier extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu); 8631d9d9b6SMarc Zyngier extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu); 87d4d5e089SThomas Gleixner extern void mask_irq(struct irq_desc *desc); 88d4d5e089SThomas Gleixner extern void unmask_irq(struct irq_desc *desc); 89328a4978SThomas Gleixner extern void unmask_threaded_irq(struct irq_desc *desc); 9046999238SThomas Gleixner 91f63b6a05SThomas Gleixner #ifdef CONFIG_SPARSE_IRQ 92f63b6a05SThomas Gleixner static inline void irq_mark_irq(unsigned int irq) { } 93f63b6a05SThomas Gleixner #else 94f63b6a05SThomas Gleixner extern void irq_mark_irq(unsigned int irq); 95f63b6a05SThomas Gleixner #endif 96f63b6a05SThomas Gleixner 9785ac16d0SYinghai Lu extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr); 980fa0ebbfSMike Travis 99edd14cfeSKeith Busch irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc, unsigned int *flags); 10071f64340SHuang Shijie irqreturn_t handle_irq_event_percpu(struct irq_desc *desc); 1014912609fSThomas Gleixner irqreturn_t handle_irq_event(struct irq_desc *desc); 1024912609fSThomas Gleixner 103e144710bSThomas Gleixner /* Resending of interrupts :*/ 1040798abebSJiang Liu void check_irq_resend(struct irq_desc *desc); 105fe200ae4SThomas Gleixner bool irq_wait_for_poll(struct irq_desc *desc); 106a92444c6SThomas Gleixner void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action); 107e144710bSThomas Gleixner 1081da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS 1092c6927a3SYinghai Lu extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); 11013bfe99eSThomas Gleixner extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); 1111da177e4SLinus Torvalds extern void register_handler_proc(unsigned int irq, struct irqaction *action); 1121da177e4SLinus Torvalds extern void unregister_handler_proc(unsigned int irq, struct irqaction *action); 1131da177e4SLinus Torvalds #else 1142c6927a3SYinghai Lu static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { } 11513bfe99eSThomas Gleixner static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { } 1161da177e4SLinus Torvalds static inline void register_handler_proc(unsigned int irq, 1171da177e4SLinus Torvalds struct irqaction *action) { } 1181da177e4SLinus Torvalds static inline void unregister_handler_proc(unsigned int irq, 1191da177e4SLinus Torvalds struct irqaction *action) { } 1201da177e4SLinus Torvalds #endif 1211da177e4SLinus Torvalds 1229c255583SThomas Gleixner extern bool irq_can_set_affinity_usr(unsigned int irq); 1239c255583SThomas Gleixner 124cba4235eSThomas Gleixner extern int irq_select_affinity_usr(unsigned int irq); 125f6d87f4bSThomas Gleixner 126591d2fb0SThomas Gleixner extern void irq_set_thread_affinity(struct irq_desc *desc); 12757b150ccSYinghai Lu 128818b0f3bSJiang Liu extern int irq_do_set_affinity(struct irq_data *data, 129818b0f3bSJiang Liu const struct cpumask *dest, bool force); 130818b0f3bSJiang Liu 13143564bd9SThomas Gleixner #ifdef CONFIG_SMP 13243564bd9SThomas Gleixner extern int irq_setup_affinity(struct irq_desc *desc); 13343564bd9SThomas Gleixner #else 13443564bd9SThomas Gleixner static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; } 13543564bd9SThomas Gleixner #endif 13643564bd9SThomas Gleixner 13770aedd24SThomas Gleixner /* Inline functions for support of irq chips on slow busses */ 1383876ec9eSThomas Gleixner static inline void chip_bus_lock(struct irq_desc *desc) 13970aedd24SThomas Gleixner { 1403876ec9eSThomas Gleixner if (unlikely(desc->irq_data.chip->irq_bus_lock)) 1413876ec9eSThomas Gleixner desc->irq_data.chip->irq_bus_lock(&desc->irq_data); 14270aedd24SThomas Gleixner } 14370aedd24SThomas Gleixner 1443876ec9eSThomas Gleixner static inline void chip_bus_sync_unlock(struct irq_desc *desc) 14570aedd24SThomas Gleixner { 1463876ec9eSThomas Gleixner if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock)) 1473876ec9eSThomas Gleixner desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data); 14870aedd24SThomas Gleixner } 14970aedd24SThomas Gleixner 15031d9d9b6SMarc Zyngier #define _IRQ_DESC_CHECK (1 << 0) 15131d9d9b6SMarc Zyngier #define _IRQ_DESC_PERCPU (1 << 1) 15231d9d9b6SMarc Zyngier 15331d9d9b6SMarc Zyngier #define IRQ_GET_DESC_CHECK_GLOBAL (_IRQ_DESC_CHECK) 15431d9d9b6SMarc Zyngier #define IRQ_GET_DESC_CHECK_PERCPU (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU) 15531d9d9b6SMarc Zyngier 156f944b5a7SDaniel Lezcano #define for_each_action_of_desc(desc, act) \ 157163616cfSMasahiro Yamada for (act = desc->action; act; act = act->next) 158f944b5a7SDaniel Lezcano 159d5eb4ad2SThomas Gleixner struct irq_desc * 16031d9d9b6SMarc Zyngier __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, 16131d9d9b6SMarc Zyngier unsigned int check); 162d5eb4ad2SThomas Gleixner void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus); 163d5eb4ad2SThomas Gleixner 164d5eb4ad2SThomas Gleixner static inline struct irq_desc * 16531d9d9b6SMarc Zyngier irq_get_desc_buslock(unsigned int irq, unsigned long *flags, unsigned int check) 166d5eb4ad2SThomas Gleixner { 16731d9d9b6SMarc Zyngier return __irq_get_desc_lock(irq, flags, true, check); 168d5eb4ad2SThomas Gleixner } 169d5eb4ad2SThomas Gleixner 170d5eb4ad2SThomas Gleixner static inline void 171d5eb4ad2SThomas Gleixner irq_put_desc_busunlock(struct irq_desc *desc, unsigned long flags) 172d5eb4ad2SThomas Gleixner { 173d5eb4ad2SThomas Gleixner __irq_put_desc_unlock(desc, flags, true); 174d5eb4ad2SThomas Gleixner } 175d5eb4ad2SThomas Gleixner 176d5eb4ad2SThomas Gleixner static inline struct irq_desc * 17731d9d9b6SMarc Zyngier irq_get_desc_lock(unsigned int irq, unsigned long *flags, unsigned int check) 178d5eb4ad2SThomas Gleixner { 17931d9d9b6SMarc Zyngier return __irq_get_desc_lock(irq, flags, false, check); 180d5eb4ad2SThomas Gleixner } 181d5eb4ad2SThomas Gleixner 182d5eb4ad2SThomas Gleixner static inline void 183d5eb4ad2SThomas Gleixner irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags) 184d5eb4ad2SThomas Gleixner { 185d5eb4ad2SThomas Gleixner __irq_put_desc_unlock(desc, flags, false); 186d5eb4ad2SThomas Gleixner } 187d5eb4ad2SThomas Gleixner 188b354286eSBoqun Feng #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) 189b354286eSBoqun Feng 190087cdfb6SThomas Gleixner static inline unsigned int irqd_get(struct irq_data *d) 191087cdfb6SThomas Gleixner { 192087cdfb6SThomas Gleixner return __irqd_to_state(d); 193087cdfb6SThomas Gleixner } 194087cdfb6SThomas Gleixner 19543f77759SIngo Molnar /* 196f230b6d5SThomas Gleixner * Manipulation functions for irq_data.state 197f230b6d5SThomas Gleixner */ 198f230b6d5SThomas Gleixner static inline void irqd_set_move_pending(struct irq_data *d) 199f230b6d5SThomas Gleixner { 2000d0b4c86SJiang Liu __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING; 201f230b6d5SThomas Gleixner } 202f230b6d5SThomas Gleixner 203f230b6d5SThomas Gleixner static inline void irqd_clr_move_pending(struct irq_data *d) 204f230b6d5SThomas Gleixner { 2050d0b4c86SJiang Liu __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING; 206f230b6d5SThomas Gleixner } 207a005677bSThomas Gleixner 20854fdf6a0SThomas Gleixner static inline void irqd_set_managed_shutdown(struct irq_data *d) 20954fdf6a0SThomas Gleixner { 21054fdf6a0SThomas Gleixner __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN; 21154fdf6a0SThomas Gleixner } 21254fdf6a0SThomas Gleixner 21354fdf6a0SThomas Gleixner static inline void irqd_clr_managed_shutdown(struct irq_data *d) 21454fdf6a0SThomas Gleixner { 21554fdf6a0SThomas Gleixner __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN; 21654fdf6a0SThomas Gleixner } 21754fdf6a0SThomas Gleixner 218a005677bSThomas Gleixner static inline void irqd_clear(struct irq_data *d, unsigned int mask) 219a005677bSThomas Gleixner { 2200d0b4c86SJiang Liu __irqd_to_state(d) &= ~mask; 221a005677bSThomas Gleixner } 222a005677bSThomas Gleixner 223a005677bSThomas Gleixner static inline void irqd_set(struct irq_data *d, unsigned int mask) 224a005677bSThomas Gleixner { 2250d0b4c86SJiang Liu __irqd_to_state(d) |= mask; 226a005677bSThomas Gleixner } 227a005677bSThomas Gleixner 2282bdd1055SThomas Gleixner static inline bool irqd_has_set(struct irq_data *d, unsigned int mask) 2292bdd1055SThomas Gleixner { 2300d0b4c86SJiang Liu return __irqd_to_state(d) & mask; 2312bdd1055SThomas Gleixner } 2328f945a33SThomas Gleixner 233a696712cSJuergen Gross static inline void irq_state_set_disabled(struct irq_desc *desc) 234a696712cSJuergen Gross { 235a696712cSJuergen Gross irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); 236a696712cSJuergen Gross } 237a696712cSJuergen Gross 238a696712cSJuergen Gross static inline void irq_state_set_masked(struct irq_desc *desc) 239a696712cSJuergen Gross { 240a696712cSJuergen Gross irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); 241a696712cSJuergen Gross } 242a696712cSJuergen Gross 243b354286eSBoqun Feng #undef __irqd_to_state 244b354286eSBoqun Feng 245*1136b072SThomas Gleixner static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc) 2468f945a33SThomas Gleixner { 2478f945a33SThomas Gleixner __this_cpu_inc(*desc->kstat_irqs); 2488f945a33SThomas Gleixner __this_cpu_inc(kstat.irqs_sum); 2498f945a33SThomas Gleixner } 250cab303beSThomas Gleixner 251*1136b072SThomas Gleixner static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc) 252*1136b072SThomas Gleixner { 253*1136b072SThomas Gleixner __kstat_incr_irqs_this_cpu(desc); 254*1136b072SThomas Gleixner desc->tot_count++; 255*1136b072SThomas Gleixner } 256*1136b072SThomas Gleixner 2576783011bSJiang Liu static inline int irq_desc_get_node(struct irq_desc *desc) 2586783011bSJiang Liu { 259449e9caeSJiang Liu return irq_common_data_get_node(&desc->irq_common_data); 2606783011bSJiang Liu } 2616783011bSJiang Liu 2624717f133SGrygorii Strashko static inline int irq_desc_is_chained(struct irq_desc *desc) 2634717f133SGrygorii Strashko { 2644717f133SGrygorii Strashko return (desc->action && desc->action == &chained_action); 2654717f133SGrygorii Strashko } 2664717f133SGrygorii Strashko 267cab303beSThomas Gleixner #ifdef CONFIG_PM_SLEEP 2689ce7a258SThomas Gleixner bool irq_pm_check_wakeup(struct irq_desc *desc); 269cab303beSThomas Gleixner void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action); 270cab303beSThomas Gleixner void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action); 271cab303beSThomas Gleixner #else 2729ce7a258SThomas Gleixner static inline bool irq_pm_check_wakeup(struct irq_desc *desc) { return false; } 273cab303beSThomas Gleixner static inline void 274cab303beSThomas Gleixner irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { } 275cab303beSThomas Gleixner static inline void 276cab303beSThomas Gleixner irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { } 277cab303beSThomas Gleixner #endif 278f1602039SBartosz Golaszewski 279b2d3d61aSDaniel Lezcano #ifdef CONFIG_IRQ_TIMINGS 280b2d3d61aSDaniel Lezcano 281b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_SHIFT 5 282b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_SIZE (1 << IRQ_TIMINGS_SHIFT) 283b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_MASK (IRQ_TIMINGS_SIZE - 1) 284b2d3d61aSDaniel Lezcano 285b2d3d61aSDaniel Lezcano /** 286b2d3d61aSDaniel Lezcano * struct irq_timings - irq timings storing structure 287b2d3d61aSDaniel Lezcano * @values: a circular buffer of u64 encoded <timestamp,irq> values 288b2d3d61aSDaniel Lezcano * @count: the number of elements in the array 289b2d3d61aSDaniel Lezcano */ 290b2d3d61aSDaniel Lezcano struct irq_timings { 291b2d3d61aSDaniel Lezcano u64 values[IRQ_TIMINGS_SIZE]; 292b2d3d61aSDaniel Lezcano int count; 293b2d3d61aSDaniel Lezcano }; 294b2d3d61aSDaniel Lezcano 295b2d3d61aSDaniel Lezcano DECLARE_PER_CPU(struct irq_timings, irq_timings); 296b2d3d61aSDaniel Lezcano 297e1c92149SDaniel Lezcano extern void irq_timings_free(int irq); 298e1c92149SDaniel Lezcano extern int irq_timings_alloc(int irq); 299e1c92149SDaniel Lezcano 300b2d3d61aSDaniel Lezcano static inline void irq_remove_timings(struct irq_desc *desc) 301b2d3d61aSDaniel Lezcano { 302b2d3d61aSDaniel Lezcano desc->istate &= ~IRQS_TIMINGS; 303e1c92149SDaniel Lezcano 304e1c92149SDaniel Lezcano irq_timings_free(irq_desc_get_irq(desc)); 305b2d3d61aSDaniel Lezcano } 306b2d3d61aSDaniel Lezcano 307b2d3d61aSDaniel Lezcano static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act) 308b2d3d61aSDaniel Lezcano { 309e1c92149SDaniel Lezcano int irq = irq_desc_get_irq(desc); 310e1c92149SDaniel Lezcano int ret; 311e1c92149SDaniel Lezcano 312b2d3d61aSDaniel Lezcano /* 313b2d3d61aSDaniel Lezcano * We don't need the measurement because the idle code already 314b2d3d61aSDaniel Lezcano * knows the next expiry event. 315b2d3d61aSDaniel Lezcano */ 316b2d3d61aSDaniel Lezcano if (act->flags & __IRQF_TIMER) 317b2d3d61aSDaniel Lezcano return; 318b2d3d61aSDaniel Lezcano 319e1c92149SDaniel Lezcano /* 320e1c92149SDaniel Lezcano * In case the timing allocation fails, we just want to warn, 321e1c92149SDaniel Lezcano * not fail, so letting the system boot anyway. 322e1c92149SDaniel Lezcano */ 323e1c92149SDaniel Lezcano ret = irq_timings_alloc(irq); 324e1c92149SDaniel Lezcano if (ret) { 325e1c92149SDaniel Lezcano pr_warn("Failed to allocate irq timing stats for irq%d (%d)", 326e1c92149SDaniel Lezcano irq, ret); 327e1c92149SDaniel Lezcano return; 328e1c92149SDaniel Lezcano } 329e1c92149SDaniel Lezcano 330b2d3d61aSDaniel Lezcano desc->istate |= IRQS_TIMINGS; 331b2d3d61aSDaniel Lezcano } 332b2d3d61aSDaniel Lezcano 333b2d3d61aSDaniel Lezcano extern void irq_timings_enable(void); 334b2d3d61aSDaniel Lezcano extern void irq_timings_disable(void); 335b2d3d61aSDaniel Lezcano 336b2d3d61aSDaniel Lezcano DECLARE_STATIC_KEY_FALSE(irq_timing_enabled); 337b2d3d61aSDaniel Lezcano 338b2d3d61aSDaniel Lezcano /* 339b2d3d61aSDaniel Lezcano * The interrupt number and the timestamp are encoded into a single 340b2d3d61aSDaniel Lezcano * u64 variable to optimize the size. 341b2d3d61aSDaniel Lezcano * 48 bit time stamp and 16 bit IRQ number is way sufficient. 342b2d3d61aSDaniel Lezcano * Who cares an IRQ after 78 hours of idle time? 343b2d3d61aSDaniel Lezcano */ 344b2d3d61aSDaniel Lezcano static inline u64 irq_timing_encode(u64 timestamp, int irq) 345b2d3d61aSDaniel Lezcano { 346b2d3d61aSDaniel Lezcano return (timestamp << 16) | irq; 347b2d3d61aSDaniel Lezcano } 348b2d3d61aSDaniel Lezcano 349b2d3d61aSDaniel Lezcano static inline int irq_timing_decode(u64 value, u64 *timestamp) 350b2d3d61aSDaniel Lezcano { 351b2d3d61aSDaniel Lezcano *timestamp = value >> 16; 352b2d3d61aSDaniel Lezcano return value & U16_MAX; 353b2d3d61aSDaniel Lezcano } 354b2d3d61aSDaniel Lezcano 355b2d3d61aSDaniel Lezcano /* 356b2d3d61aSDaniel Lezcano * The function record_irq_time is only called in one place in the 357b2d3d61aSDaniel Lezcano * interrupts handler. We want this function always inline so the code 358b2d3d61aSDaniel Lezcano * inside is embedded in the function and the static key branching 359b2d3d61aSDaniel Lezcano * code can act at the higher level. Without the explicit 360b2d3d61aSDaniel Lezcano * __always_inline we can end up with a function call and a small 361b2d3d61aSDaniel Lezcano * overhead in the hotpath for nothing. 362b2d3d61aSDaniel Lezcano */ 363b2d3d61aSDaniel Lezcano static __always_inline void record_irq_time(struct irq_desc *desc) 364b2d3d61aSDaniel Lezcano { 365b2d3d61aSDaniel Lezcano if (!static_branch_likely(&irq_timing_enabled)) 366b2d3d61aSDaniel Lezcano return; 367b2d3d61aSDaniel Lezcano 368b2d3d61aSDaniel Lezcano if (desc->istate & IRQS_TIMINGS) { 369b2d3d61aSDaniel Lezcano struct irq_timings *timings = this_cpu_ptr(&irq_timings); 370b2d3d61aSDaniel Lezcano 371b2d3d61aSDaniel Lezcano timings->values[timings->count & IRQ_TIMINGS_MASK] = 372b2d3d61aSDaniel Lezcano irq_timing_encode(local_clock(), 373b2d3d61aSDaniel Lezcano irq_desc_get_irq(desc)); 374b2d3d61aSDaniel Lezcano 375b2d3d61aSDaniel Lezcano timings->count++; 376b2d3d61aSDaniel Lezcano } 377b2d3d61aSDaniel Lezcano } 378b2d3d61aSDaniel Lezcano #else 379b2d3d61aSDaniel Lezcano static inline void irq_remove_timings(struct irq_desc *desc) {} 380b2d3d61aSDaniel Lezcano static inline void irq_setup_timings(struct irq_desc *desc, 381b2d3d61aSDaniel Lezcano struct irqaction *act) {}; 382b2d3d61aSDaniel Lezcano static inline void record_irq_time(struct irq_desc *desc) {} 383b2d3d61aSDaniel Lezcano #endif /* CONFIG_IRQ_TIMINGS */ 384b2d3d61aSDaniel Lezcano 385b2d3d61aSDaniel Lezcano 386f1602039SBartosz Golaszewski #ifdef CONFIG_GENERIC_IRQ_CHIP 387f1602039SBartosz Golaszewski void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, 388f1602039SBartosz Golaszewski int num_ct, unsigned int irq_base, 389f1602039SBartosz Golaszewski void __iomem *reg_base, irq_flow_handler_t handler); 390f1602039SBartosz Golaszewski #else 391f1602039SBartosz Golaszewski static inline void 392f1602039SBartosz Golaszewski irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, 393f1602039SBartosz Golaszewski int num_ct, unsigned int irq_base, 394f1602039SBartosz Golaszewski void __iomem *reg_base, irq_flow_handler_t handler) { } 395f1602039SBartosz Golaszewski #endif /* CONFIG_GENERIC_IRQ_CHIP */ 396087cdfb6SThomas Gleixner 397137221dfSChristoph Hellwig #ifdef CONFIG_GENERIC_PENDING_IRQ 398137221dfSChristoph Hellwig static inline bool irq_can_move_pcntxt(struct irq_data *data) 399137221dfSChristoph Hellwig { 400137221dfSChristoph Hellwig return irqd_can_move_in_process_context(data); 401137221dfSChristoph Hellwig } 402137221dfSChristoph Hellwig static inline bool irq_move_pending(struct irq_data *data) 403137221dfSChristoph Hellwig { 404137221dfSChristoph Hellwig return irqd_is_setaffinity_pending(data); 405137221dfSChristoph Hellwig } 406137221dfSChristoph Hellwig static inline void 407137221dfSChristoph Hellwig irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) 408137221dfSChristoph Hellwig { 409137221dfSChristoph Hellwig cpumask_copy(desc->pending_mask, mask); 410137221dfSChristoph Hellwig } 411137221dfSChristoph Hellwig static inline void 412137221dfSChristoph Hellwig irq_get_pending(struct cpumask *mask, struct irq_desc *desc) 413137221dfSChristoph Hellwig { 414137221dfSChristoph Hellwig cpumask_copy(mask, desc->pending_mask); 415137221dfSChristoph Hellwig } 416f0383c24SThomas Gleixner static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc) 417f0383c24SThomas Gleixner { 418f0383c24SThomas Gleixner return desc->pending_mask; 419f0383c24SThomas Gleixner } 42036d84fb4SThomas Gleixner bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear); 421137221dfSChristoph Hellwig #else /* CONFIG_GENERIC_PENDING_IRQ */ 422137221dfSChristoph Hellwig static inline bool irq_can_move_pcntxt(struct irq_data *data) 423137221dfSChristoph Hellwig { 424137221dfSChristoph Hellwig return true; 425137221dfSChristoph Hellwig } 426137221dfSChristoph Hellwig static inline bool irq_move_pending(struct irq_data *data) 427137221dfSChristoph Hellwig { 428137221dfSChristoph Hellwig return false; 429137221dfSChristoph Hellwig } 430137221dfSChristoph Hellwig static inline void 431137221dfSChristoph Hellwig irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) 432137221dfSChristoph Hellwig { 433137221dfSChristoph Hellwig } 434137221dfSChristoph Hellwig static inline void 435137221dfSChristoph Hellwig irq_get_pending(struct cpumask *mask, struct irq_desc *desc) 436137221dfSChristoph Hellwig { 437137221dfSChristoph Hellwig } 438f0383c24SThomas Gleixner static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc) 439f0383c24SThomas Gleixner { 440f0383c24SThomas Gleixner return NULL; 441f0383c24SThomas Gleixner } 44236d84fb4SThomas Gleixner static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear) 44336d84fb4SThomas Gleixner { 44436d84fb4SThomas Gleixner return false; 44536d84fb4SThomas Gleixner } 446f0383c24SThomas Gleixner #endif /* !CONFIG_GENERIC_PENDING_IRQ */ 447137221dfSChristoph Hellwig 448457f6d35SThomas Gleixner #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY) 449702cb0a0SThomas Gleixner static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve) 450457f6d35SThomas Gleixner { 451457f6d35SThomas Gleixner irqd_set_activated(data); 452bb9b428aSThomas Gleixner return 0; 453457f6d35SThomas Gleixner } 454457f6d35SThomas Gleixner static inline void irq_domain_deactivate_irq(struct irq_data *data) 455457f6d35SThomas Gleixner { 456457f6d35SThomas Gleixner irqd_clr_activated(data); 457457f6d35SThomas Gleixner } 458457f6d35SThomas Gleixner #endif 459457f6d35SThomas Gleixner 460087cdfb6SThomas Gleixner #ifdef CONFIG_GENERIC_IRQ_DEBUGFS 461c2ce34c0SThomas Gleixner #include <linux/debugfs.h> 462c2ce34c0SThomas Gleixner 463087cdfb6SThomas Gleixner void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc); 464c2ce34c0SThomas Gleixner static inline void irq_remove_debugfs_entry(struct irq_desc *desc) 465c2ce34c0SThomas Gleixner { 466c2ce34c0SThomas Gleixner debugfs_remove(desc->debugfs_file); 46707557ccbSThomas Gleixner kfree(desc->dev_name); 468c2ce34c0SThomas Gleixner } 46907557ccbSThomas Gleixner void irq_debugfs_copy_devname(int irq, struct device *dev); 470087cdfb6SThomas Gleixner # ifdef CONFIG_IRQ_DOMAIN 471087cdfb6SThomas Gleixner void irq_domain_debugfs_init(struct dentry *root); 472087cdfb6SThomas Gleixner # else 473e5682b4eSSebastian Ott static inline void irq_domain_debugfs_init(struct dentry *root) 474e5682b4eSSebastian Ott { 475e5682b4eSSebastian Ott } 476087cdfb6SThomas Gleixner # endif 477087cdfb6SThomas Gleixner #else /* CONFIG_GENERIC_IRQ_DEBUGFS */ 478087cdfb6SThomas Gleixner static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d) 479087cdfb6SThomas Gleixner { 480087cdfb6SThomas Gleixner } 481087cdfb6SThomas Gleixner static inline void irq_remove_debugfs_entry(struct irq_desc *d) 482087cdfb6SThomas Gleixner { 483087cdfb6SThomas Gleixner } 48407557ccbSThomas Gleixner static inline void irq_debugfs_copy_devname(int irq, struct device *dev) 48507557ccbSThomas Gleixner { 48607557ccbSThomas Gleixner } 487087cdfb6SThomas Gleixner #endif /* CONFIG_GENERIC_IRQ_DEBUGFS */ 488