1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple SoC CPU cluster performance state driver 4 * 5 * Copyright The Asahi Linux Contributors 6 * 7 * Based on scpi-cpufreq.c 8 */ 9 10 #include <linux/bitfield.h> 11 #include <linux/bitops.h> 12 #include <linux/cpu.h> 13 #include <linux/cpufreq.h> 14 #include <linux/cpumask.h> 15 #include <linux/delay.h> 16 #include <linux/err.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_address.h> 22 #include <linux/pm_opp.h> 23 #include <linux/slab.h> 24 25 #define APPLE_DVFS_CMD 0x20 26 #define APPLE_DVFS_CMD_BUSY BIT(31) 27 #define APPLE_DVFS_CMD_SET BIT(25) 28 #define APPLE_DVFS_CMD_PS2 GENMASK(16, 12) 29 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) 30 31 /* Same timebase as CPU counter (24MHz) */ 32 #define APPLE_DVFS_LAST_CHG_TIME 0x38 33 34 /* 35 * Apple ran out of bits and had to shift this in T8112... 36 */ 37 #define APPLE_DVFS_STATUS 0x50 38 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) 39 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 40 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) 41 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) 42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 43 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) 44 45 /* 46 * Div is +1, base clock is 12MHz on existing SoCs. 47 * For documentation purposes. We use the OPP table to 48 * get the frequency. 49 */ 50 #define APPLE_DVFS_PLL_STATUS 0xc0 51 #define APPLE_DVFS_PLL_FACTOR 0xc8 52 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) 53 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) 54 55 #define APPLE_DVFS_TRANSITION_TIMEOUT 100 56 57 struct apple_soc_cpufreq_info { 58 u64 max_pstate; 59 u64 cur_pstate_mask; 60 u64 cur_pstate_shift; 61 }; 62 63 struct apple_cpu_priv { 64 struct device *cpu_dev; 65 void __iomem *reg_base; 66 const struct apple_soc_cpufreq_info *info; 67 }; 68 69 static struct cpufreq_driver apple_soc_cpufreq_driver; 70 71 static const struct apple_soc_cpufreq_info soc_t8103_info = { 72 .max_pstate = 15, 73 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, 74 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, 75 }; 76 77 static const struct apple_soc_cpufreq_info soc_t8112_info = { 78 .max_pstate = 31, 79 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, 80 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, 81 }; 82 83 static const struct apple_soc_cpufreq_info soc_default_info = { 84 .max_pstate = 15, 85 .cur_pstate_mask = 0, /* fallback */ 86 }; 87 88 static const struct of_device_id apple_soc_cpufreq_of_match[] = { 89 { 90 .compatible = "apple,t8103-cluster-cpufreq", 91 .data = &soc_t8103_info, 92 }, 93 { 94 .compatible = "apple,t8112-cluster-cpufreq", 95 .data = &soc_t8112_info, 96 }, 97 { 98 .compatible = "apple,cluster-cpufreq", 99 .data = &soc_default_info, 100 }, 101 {} 102 }; 103 104 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) 105 { 106 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 107 struct apple_cpu_priv *priv = policy->driver_data; 108 struct cpufreq_frequency_table *p; 109 unsigned int pstate; 110 111 if (priv->info->cur_pstate_mask) { 112 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS); 113 114 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; 115 } else { 116 /* 117 * For the fallback case we might not know the layout of DVFS_STATUS, 118 * so just use the command register value (which ignores boost limitations). 119 */ 120 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); 121 122 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); 123 } 124 125 cpufreq_for_each_valid_entry(p, policy->freq_table) 126 if (p->driver_data == pstate) 127 return p->frequency; 128 129 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", 130 pstate); 131 return 0; 132 } 133 134 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 135 unsigned int index) 136 { 137 struct apple_cpu_priv *priv = policy->driver_data; 138 unsigned int pstate = policy->freq_table[index].driver_data; 139 u64 reg; 140 141 /* Fallback for newer SoCs */ 142 if (index > priv->info->max_pstate) 143 index = priv->info->max_pstate; 144 145 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, 146 !(reg & APPLE_DVFS_CMD_BUSY), 2, 147 APPLE_DVFS_TRANSITION_TIMEOUT)) { 148 return -EIO; 149 } 150 151 reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2); 152 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate); 153 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); 154 reg |= APPLE_DVFS_CMD_SET; 155 156 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); 157 158 return 0; 159 } 160 161 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, 162 unsigned int target_freq) 163 { 164 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) 165 return 0; 166 167 return policy->freq_table[policy->cached_resolved_idx].frequency; 168 } 169 170 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, 171 void __iomem **reg_base, 172 const struct apple_soc_cpufreq_info **info) 173 { 174 struct of_phandle_args args; 175 const struct of_device_id *match; 176 int ret = 0; 177 178 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", 179 "#performance-domain-cells", 180 policy->cpus, &args); 181 if (ret < 0) 182 return ret; 183 184 match = of_match_node(apple_soc_cpufreq_of_match, args.np); 185 of_node_put(args.np); 186 if (!match) 187 return -ENODEV; 188 189 *info = match->data; 190 191 *reg_base = of_iomap(args.np, 0); 192 if (IS_ERR(*reg_base)) 193 return PTR_ERR(*reg_base); 194 195 return 0; 196 } 197 198 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = { 199 &cpufreq_freq_attr_scaling_available_freqs, 200 NULL, /* Filled in below if boost is enabled */ 201 NULL, 202 }; 203 204 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) 205 { 206 int ret, i; 207 unsigned int transition_latency; 208 void __iomem *reg_base; 209 struct device *cpu_dev; 210 struct apple_cpu_priv *priv; 211 const struct apple_soc_cpufreq_info *info; 212 struct cpufreq_frequency_table *freq_table; 213 214 cpu_dev = get_cpu_device(policy->cpu); 215 if (!cpu_dev) { 216 pr_err("failed to get cpu%d device\n", policy->cpu); 217 return -ENODEV; 218 } 219 220 ret = dev_pm_opp_of_add_table(cpu_dev); 221 if (ret < 0) { 222 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); 223 return ret; 224 } 225 226 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); 227 if (ret) { 228 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); 229 return ret; 230 } 231 232 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 233 if (ret) { 234 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); 235 goto out_iounmap; 236 } 237 238 ret = dev_pm_opp_get_opp_count(cpu_dev); 239 if (ret <= 0) { 240 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); 241 ret = -EPROBE_DEFER; 242 goto out_free_opp; 243 } 244 245 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 246 if (!priv) { 247 ret = -ENOMEM; 248 goto out_free_opp; 249 } 250 251 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 252 if (ret) { 253 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 254 goto out_free_priv; 255 } 256 257 /* Get OPP levels (p-state indexes) and stash them in driver_data */ 258 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 259 unsigned long rate = freq_table[i].frequency * 1000 + 999; 260 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); 261 262 if (IS_ERR(opp)) { 263 ret = PTR_ERR(opp); 264 goto out_free_cpufreq_table; 265 } 266 freq_table[i].driver_data = dev_pm_opp_get_level(opp); 267 dev_pm_opp_put(opp); 268 } 269 270 priv->cpu_dev = cpu_dev; 271 priv->reg_base = reg_base; 272 priv->info = info; 273 policy->driver_data = priv; 274 policy->freq_table = freq_table; 275 276 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); 277 if (!transition_latency) 278 transition_latency = CPUFREQ_ETERNAL; 279 280 policy->cpuinfo.transition_latency = transition_latency; 281 policy->dvfs_possible_from_any_cpu = true; 282 policy->fast_switch_possible = true; 283 284 if (policy_has_boost_freq(policy)) { 285 ret = cpufreq_enable_boost_support(); 286 if (ret) { 287 dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 288 } else { 289 apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; 290 apple_soc_cpufreq_driver.boost_enabled = true; 291 } 292 } 293 294 return 0; 295 296 out_free_cpufreq_table: 297 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); 298 out_free_priv: 299 kfree(priv); 300 out_free_opp: 301 dev_pm_opp_remove_all_dynamic(cpu_dev); 302 out_iounmap: 303 iounmap(reg_base); 304 return ret; 305 } 306 307 static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy) 308 { 309 struct apple_cpu_priv *priv = policy->driver_data; 310 311 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); 312 dev_pm_opp_remove_all_dynamic(priv->cpu_dev); 313 iounmap(priv->reg_base); 314 kfree(priv); 315 316 return 0; 317 } 318 319 static struct cpufreq_driver apple_soc_cpufreq_driver = { 320 .name = "apple-cpufreq", 321 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 322 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, 323 .verify = cpufreq_generic_frequency_table_verify, 324 .attr = cpufreq_generic_attr, 325 .get = apple_soc_cpufreq_get_rate, 326 .init = apple_soc_cpufreq_init, 327 .exit = apple_soc_cpufreq_exit, 328 .target_index = apple_soc_cpufreq_set_target, 329 .fast_switch = apple_soc_cpufreq_fast_switch, 330 .register_em = cpufreq_register_em_with_opp, 331 .attr = apple_soc_cpufreq_hw_attr, 332 }; 333 334 static int __init apple_soc_cpufreq_module_init(void) 335 { 336 if (!of_machine_is_compatible("apple,arm-platform")) 337 return -ENODEV; 338 339 return cpufreq_register_driver(&apple_soc_cpufreq_driver); 340 } 341 module_init(apple_soc_cpufreq_module_init); 342 343 static void __exit apple_soc_cpufreq_module_exit(void) 344 { 345 cpufreq_unregister_driver(&apple_soc_cpufreq_driver); 346 } 347 module_exit(apple_soc_cpufreq_module_exit); 348 349 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); 350 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); 351 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); 352 MODULE_LICENSE("GPL"); 353