1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/wait_bit.h> 8 9 #include "intel_runtime_pm.h" 10 #include "intel_wakeref.h" 11 #include "i915_drv.h" 12 13 static void rpm_get(struct intel_wakeref *wf) 14 { 15 wf->wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm); 16 } 17 18 static void rpm_put(struct intel_wakeref *wf) 19 { 20 intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref); 21 22 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); 23 INTEL_WAKEREF_BUG_ON(!wakeref); 24 } 25 26 int __intel_wakeref_get_first(struct intel_wakeref *wf) 27 { 28 /* 29 * Treat get/put as different subclasses, as we may need to run 30 * the put callback from under the shrinker and do not want to 31 * cross-contanimate that callback with any extra work performed 32 * upon acquiring the wakeref. 33 */ 34 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING); 35 if (!atomic_read(&wf->count)) { 36 int err; 37 38 rpm_get(wf); 39 40 err = wf->ops->get(wf); 41 if (unlikely(err)) { 42 rpm_put(wf); 43 mutex_unlock(&wf->mutex); 44 return err; 45 } 46 47 smp_mb__before_atomic(); /* release wf->count */ 48 } 49 atomic_inc(&wf->count); 50 mutex_unlock(&wf->mutex); 51 52 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); 53 return 0; 54 } 55 56 static void ____intel_wakeref_put_last(struct intel_wakeref *wf) 57 { 58 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); 59 if (unlikely(!atomic_dec_and_test(&wf->count))) 60 goto unlock; 61 62 /* ops->put() must reschedule its own release on error/deferral */ 63 if (likely(!wf->ops->put(wf))) { 64 rpm_put(wf); 65 wake_up_var(&wf->wakeref); 66 } 67 68 unlock: 69 mutex_unlock(&wf->mutex); 70 } 71 72 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) 73 { 74 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); 75 76 /* Assume we are not in process context and so cannot sleep. */ 77 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { 78 mod_delayed_work(wf->i915->unordered_wq, &wf->work, 79 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); 80 return; 81 } 82 83 ____intel_wakeref_put_last(wf); 84 } 85 86 static void __intel_wakeref_put_work(struct work_struct *wrk) 87 { 88 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); 89 90 if (atomic_add_unless(&wf->count, -1, 1)) 91 return; 92 93 mutex_lock(&wf->mutex); 94 ____intel_wakeref_put_last(wf); 95 } 96 97 void __intel_wakeref_init(struct intel_wakeref *wf, 98 struct drm_i915_private *i915, 99 const struct intel_wakeref_ops *ops, 100 struct intel_wakeref_lockclass *key) 101 { 102 wf->i915 = i915; 103 wf->ops = ops; 104 105 #ifdef __linux__ 106 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex); 107 #else 108 rw_init(&wf->mutex, "wakeref.mutex"); 109 #endif 110 atomic_set(&wf->count, 0); 111 wf->wakeref = 0; 112 113 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); 114 lockdep_init_map(&wf->work.work.lockdep_map, 115 "wakeref.work", &key->work, 0); 116 } 117 118 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) 119 { 120 int err; 121 122 might_sleep(); 123 124 err = wait_var_event_killable(&wf->wakeref, 125 !intel_wakeref_is_active(wf)); 126 if (err) 127 return err; 128 129 intel_wakeref_unlock_wait(wf); 130 return 0; 131 } 132 133 static void wakeref_auto_timeout(void *arg) 134 { 135 struct intel_wakeref_auto *wf = arg; 136 intel_wakeref_t wakeref; 137 unsigned long flags; 138 139 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags)) 140 return; 141 142 wakeref = fetch_and_zero(&wf->wakeref); 143 spin_unlock_irqrestore(&wf->lock, flags); 144 145 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); 146 } 147 148 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, 149 struct drm_i915_private *i915) 150 { 151 mtx_init(&wf->lock, IPL_TTY); 152 #ifdef __linux__ 153 timer_setup(&wf->timer, wakeref_auto_timeout, 0); 154 #else 155 timeout_set(&wf->timer, wakeref_auto_timeout, wf); 156 #endif 157 refcount_set(&wf->count, 0); 158 wf->wakeref = 0; 159 wf->i915 = i915; 160 } 161 162 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) 163 { 164 unsigned long flags; 165 166 if (!timeout) { 167 if (del_timer_sync(&wf->timer)) 168 wakeref_auto_timeout(&wf->timer); 169 return; 170 } 171 172 /* Our mission is that we only extend an already active wakeref */ 173 assert_rpm_wakelock_held(&wf->i915->runtime_pm); 174 175 if (!refcount_inc_not_zero(&wf->count)) { 176 spin_lock_irqsave(&wf->lock, flags); 177 if (!refcount_inc_not_zero(&wf->count)) { 178 INTEL_WAKEREF_BUG_ON(wf->wakeref); 179 wf->wakeref = 180 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm); 181 refcount_set(&wf->count, 1); 182 } 183 spin_unlock_irqrestore(&wf->lock, flags); 184 } 185 186 /* 187 * If we extend a pending timer, we will only get a single timer 188 * callback and so need to cancel the local inc by running the 189 * elided callback to keep the wf->count balanced. 190 */ 191 if (mod_timer(&wf->timer, jiffies + timeout)) 192 wakeref_auto_timeout(&wf->timer); 193 } 194 195 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) 196 { 197 intel_wakeref_auto(wf, 0); 198 INTEL_WAKEREF_BUG_ON(wf->wakeref); 199 } 200