17017858eSGreentime Hu // SPDX-License-Identifier: GPL-2.0-or-later
27017858eSGreentime Hu /*
37017858eSGreentime Hu * Copyright (C) 2023 SiFive
47017858eSGreentime Hu * Author: Andy Chiu <andy.chiu@sifive.com>
57017858eSGreentime Hu */
67017858eSGreentime Hu #include <linux/export.h>
7cd054837SAndy Chiu #include <linux/sched/signal.h>
8cd054837SAndy Chiu #include <linux/types.h>
9cd054837SAndy Chiu #include <linux/slab.h>
10cd054837SAndy Chiu #include <linux/sched.h>
11cd054837SAndy Chiu #include <linux/uaccess.h>
121fd96a3eSAndy Chiu #include <linux/prctl.h>
137017858eSGreentime Hu
14cd054837SAndy Chiu #include <asm/thread_info.h>
15cd054837SAndy Chiu #include <asm/processor.h>
16cd054837SAndy Chiu #include <asm/insn.h>
177017858eSGreentime Hu #include <asm/vector.h>
187017858eSGreentime Hu #include <asm/csr.h>
197017858eSGreentime Hu #include <asm/elf.h>
20cd054837SAndy Chiu #include <asm/ptrace.h>
217017858eSGreentime Hu #include <asm/bug.h>
227017858eSGreentime Hu
231fd96a3eSAndy Chiu static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
24bd446f5dSAndy Chiu static struct kmem_cache *riscv_v_user_cachep;
25*2080ff94SAndy Chiu #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
26*2080ff94SAndy Chiu static struct kmem_cache *riscv_v_kernel_cachep;
27*2080ff94SAndy Chiu #endif
281fd96a3eSAndy Chiu
297017858eSGreentime Hu unsigned long riscv_v_vsize __read_mostly;
307017858eSGreentime Hu EXPORT_SYMBOL_GPL(riscv_v_vsize);
317017858eSGreentime Hu
riscv_v_setup_vsize(void)327017858eSGreentime Hu int riscv_v_setup_vsize(void)
337017858eSGreentime Hu {
347017858eSGreentime Hu unsigned long this_vsize;
357017858eSGreentime Hu
367017858eSGreentime Hu /* There are 32 vector registers with vlenb length. */
377017858eSGreentime Hu riscv_v_enable();
387017858eSGreentime Hu this_vsize = csr_read(CSR_VLENB) * 32;
397017858eSGreentime Hu riscv_v_disable();
407017858eSGreentime Hu
417017858eSGreentime Hu if (!riscv_v_vsize) {
427017858eSGreentime Hu riscv_v_vsize = this_vsize;
437017858eSGreentime Hu return 0;
447017858eSGreentime Hu }
457017858eSGreentime Hu
467017858eSGreentime Hu if (riscv_v_vsize != this_vsize) {
477017858eSGreentime Hu WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
487017858eSGreentime Hu return -EOPNOTSUPP;
497017858eSGreentime Hu }
507017858eSGreentime Hu
517017858eSGreentime Hu return 0;
527017858eSGreentime Hu }
53cd054837SAndy Chiu
riscv_v_setup_ctx_cache(void)54bd446f5dSAndy Chiu void __init riscv_v_setup_ctx_cache(void)
55bd446f5dSAndy Chiu {
56bd446f5dSAndy Chiu if (!has_vector())
57bd446f5dSAndy Chiu return;
58bd446f5dSAndy Chiu
59bd446f5dSAndy Chiu riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx",
60bd446f5dSAndy Chiu riscv_v_vsize, 16, SLAB_PANIC,
61bd446f5dSAndy Chiu 0, riscv_v_vsize, NULL);
62*2080ff94SAndy Chiu #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
63*2080ff94SAndy Chiu riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
64*2080ff94SAndy Chiu riscv_v_vsize, 16,
65*2080ff94SAndy Chiu SLAB_PANIC, NULL);
66*2080ff94SAndy Chiu #endif
67bd446f5dSAndy Chiu }
68bd446f5dSAndy Chiu
insn_is_vector(u32 insn_buf)69cd054837SAndy Chiu static bool insn_is_vector(u32 insn_buf)
70cd054837SAndy Chiu {
71cd054837SAndy Chiu u32 opcode = insn_buf & __INSN_OPCODE_MASK;
72cd054837SAndy Chiu u32 width, csr;
73cd054837SAndy Chiu
74cd054837SAndy Chiu /*
75cd054837SAndy Chiu * All V-related instructions, including CSR operations are 4-Byte. So,
76cd054837SAndy Chiu * do not handle if the instruction length is not 4-Byte.
77cd054837SAndy Chiu */
78cd054837SAndy Chiu if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
79cd054837SAndy Chiu return false;
80cd054837SAndy Chiu
81cd054837SAndy Chiu switch (opcode) {
82cd054837SAndy Chiu case RVV_OPCODE_VECTOR:
83cd054837SAndy Chiu return true;
84cd054837SAndy Chiu case RVV_OPCODE_VL:
85cd054837SAndy Chiu case RVV_OPCODE_VS:
86cd054837SAndy Chiu width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
87cd054837SAndy Chiu if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
88cd054837SAndy Chiu width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
89cd054837SAndy Chiu return true;
90cd054837SAndy Chiu
91cd054837SAndy Chiu break;
92cd054837SAndy Chiu case RVG_OPCODE_SYSTEM:
93cd054837SAndy Chiu csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
94cd054837SAndy Chiu if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
95cd054837SAndy Chiu (csr >= CSR_VL && csr <= CSR_VLENB))
96cd054837SAndy Chiu return true;
97cd054837SAndy Chiu }
98cd054837SAndy Chiu
99cd054837SAndy Chiu return false;
100cd054837SAndy Chiu }
101cd054837SAndy Chiu
riscv_v_thread_zalloc(struct kmem_cache * cache,struct __riscv_v_ext_state * ctx)102*2080ff94SAndy Chiu static int riscv_v_thread_zalloc(struct kmem_cache *cache,
103*2080ff94SAndy Chiu struct __riscv_v_ext_state *ctx)
104cd054837SAndy Chiu {
105cd054837SAndy Chiu void *datap;
106cd054837SAndy Chiu
107*2080ff94SAndy Chiu datap = kmem_cache_zalloc(cache, GFP_KERNEL);
108cd054837SAndy Chiu if (!datap)
109cd054837SAndy Chiu return -ENOMEM;
110cd054837SAndy Chiu
111*2080ff94SAndy Chiu ctx->datap = datap;
112*2080ff94SAndy Chiu memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
113cd054837SAndy Chiu return 0;
114cd054837SAndy Chiu }
115cd054837SAndy Chiu
riscv_v_thread_alloc(struct task_struct * tsk)116*2080ff94SAndy Chiu void riscv_v_thread_alloc(struct task_struct *tsk)
117*2080ff94SAndy Chiu {
118*2080ff94SAndy Chiu #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
119*2080ff94SAndy Chiu riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
120*2080ff94SAndy Chiu #endif
121*2080ff94SAndy Chiu }
122*2080ff94SAndy Chiu
riscv_v_thread_free(struct task_struct * tsk)123bd446f5dSAndy Chiu void riscv_v_thread_free(struct task_struct *tsk)
124bd446f5dSAndy Chiu {
125bd446f5dSAndy Chiu if (tsk->thread.vstate.datap)
126bd446f5dSAndy Chiu kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);
127*2080ff94SAndy Chiu #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
128*2080ff94SAndy Chiu if (tsk->thread.kernel_vstate.datap)
129*2080ff94SAndy Chiu kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
130*2080ff94SAndy Chiu #endif
131bd446f5dSAndy Chiu }
132bd446f5dSAndy Chiu
1331fd96a3eSAndy Chiu #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
1341fd96a3eSAndy Chiu #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
1351fd96a3eSAndy Chiu #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
1361fd96a3eSAndy Chiu #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
riscv_v_ctrl_get_cur(struct task_struct * tsk)1371fd96a3eSAndy Chiu static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
1381fd96a3eSAndy Chiu {
1391fd96a3eSAndy Chiu return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
1401fd96a3eSAndy Chiu }
1411fd96a3eSAndy Chiu
riscv_v_ctrl_get_next(struct task_struct * tsk)1421fd96a3eSAndy Chiu static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
1431fd96a3eSAndy Chiu {
1441fd96a3eSAndy Chiu return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
1451fd96a3eSAndy Chiu }
1461fd96a3eSAndy Chiu
riscv_v_ctrl_test_inherit(struct task_struct * tsk)1471fd96a3eSAndy Chiu static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
1481fd96a3eSAndy Chiu {
1491fd96a3eSAndy Chiu return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
1501fd96a3eSAndy Chiu }
1511fd96a3eSAndy Chiu
riscv_v_ctrl_set(struct task_struct * tsk,int cur,int nxt,bool inherit)1521fd96a3eSAndy Chiu static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
1531fd96a3eSAndy Chiu bool inherit)
1541fd96a3eSAndy Chiu {
1551fd96a3eSAndy Chiu unsigned long ctrl;
1561fd96a3eSAndy Chiu
1571fd96a3eSAndy Chiu ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
1581fd96a3eSAndy Chiu ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
1591fd96a3eSAndy Chiu if (inherit)
1601fd96a3eSAndy Chiu ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
1615b6048f2SAndy Chiu tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
1625b6048f2SAndy Chiu tsk->thread.vstate_ctrl |= ctrl;
1631fd96a3eSAndy Chiu }
1641fd96a3eSAndy Chiu
riscv_v_vstate_ctrl_user_allowed(void)1651fd96a3eSAndy Chiu bool riscv_v_vstate_ctrl_user_allowed(void)
1661fd96a3eSAndy Chiu {
1671fd96a3eSAndy Chiu return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
1681fd96a3eSAndy Chiu }
1691fd96a3eSAndy Chiu EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
1701fd96a3eSAndy Chiu
riscv_v_first_use_handler(struct pt_regs * regs)171cd054837SAndy Chiu bool riscv_v_first_use_handler(struct pt_regs *regs)
172cd054837SAndy Chiu {
173cd054837SAndy Chiu u32 __user *epc = (u32 __user *)regs->epc;
174cd054837SAndy Chiu u32 insn = (u32)regs->badaddr;
175cd054837SAndy Chiu
176cd054837SAndy Chiu /* Do not handle if V is not supported, or disabled */
177cd054837SAndy Chiu if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
178cd054837SAndy Chiu return false;
179cd054837SAndy Chiu
180cd054837SAndy Chiu /* If V has been enabled then it is not the first-use trap */
181cd054837SAndy Chiu if (riscv_v_vstate_query(regs))
182cd054837SAndy Chiu return false;
183cd054837SAndy Chiu
184cd054837SAndy Chiu /* Get the instruction */
185cd054837SAndy Chiu if (!insn) {
186cd054837SAndy Chiu if (__get_user(insn, epc))
187cd054837SAndy Chiu return false;
188cd054837SAndy Chiu }
189cd054837SAndy Chiu
190cd054837SAndy Chiu /* Filter out non-V instructions */
191cd054837SAndy Chiu if (!insn_is_vector(insn))
192cd054837SAndy Chiu return false;
193cd054837SAndy Chiu
194cd054837SAndy Chiu /* Sanity check. datap should be null by the time of the first-use trap */
195cd054837SAndy Chiu WARN_ON(current->thread.vstate.datap);
196cd054837SAndy Chiu
197cd054837SAndy Chiu /*
198cd054837SAndy Chiu * Now we sure that this is a V instruction. And it executes in the
199cd054837SAndy Chiu * context where VS has been off. So, try to allocate the user's V
200cd054837SAndy Chiu * context and resume execution.
201cd054837SAndy Chiu */
202*2080ff94SAndy Chiu if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) {
203cd054837SAndy Chiu force_sig(SIGBUS);
204cd054837SAndy Chiu return true;
205cd054837SAndy Chiu }
206cd054837SAndy Chiu riscv_v_vstate_on(regs);
2077df56cbcSAndy Chiu riscv_v_vstate_set_restore(current, regs);
208cd054837SAndy Chiu return true;
209cd054837SAndy Chiu }
2101fd96a3eSAndy Chiu
riscv_v_vstate_ctrl_init(struct task_struct * tsk)2111fd96a3eSAndy Chiu void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
2121fd96a3eSAndy Chiu {
2131fd96a3eSAndy Chiu bool inherit;
2141fd96a3eSAndy Chiu int cur, next;
2151fd96a3eSAndy Chiu
2161fd96a3eSAndy Chiu if (!has_vector())
2171fd96a3eSAndy Chiu return;
2181fd96a3eSAndy Chiu
2191fd96a3eSAndy Chiu next = riscv_v_ctrl_get_next(tsk);
2201fd96a3eSAndy Chiu if (!next) {
2217ca7a7b9SAndy Chiu if (READ_ONCE(riscv_v_implicit_uacc))
2221fd96a3eSAndy Chiu cur = PR_RISCV_V_VSTATE_CTRL_ON;
2231fd96a3eSAndy Chiu else
2241fd96a3eSAndy Chiu cur = PR_RISCV_V_VSTATE_CTRL_OFF;
2251fd96a3eSAndy Chiu } else {
2261fd96a3eSAndy Chiu cur = next;
2271fd96a3eSAndy Chiu }
2281fd96a3eSAndy Chiu /* Clear next mask if inherit-bit is not set */
2291fd96a3eSAndy Chiu inherit = riscv_v_ctrl_test_inherit(tsk);
2301fd96a3eSAndy Chiu if (!inherit)
2311fd96a3eSAndy Chiu next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
2321fd96a3eSAndy Chiu
2331fd96a3eSAndy Chiu riscv_v_ctrl_set(tsk, cur, next, inherit);
2341fd96a3eSAndy Chiu }
2351fd96a3eSAndy Chiu
riscv_v_vstate_ctrl_get_current(void)2361fd96a3eSAndy Chiu long riscv_v_vstate_ctrl_get_current(void)
2371fd96a3eSAndy Chiu {
2381fd96a3eSAndy Chiu if (!has_vector())
2391fd96a3eSAndy Chiu return -EINVAL;
2401fd96a3eSAndy Chiu
2411fd96a3eSAndy Chiu return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
2421fd96a3eSAndy Chiu }
2431fd96a3eSAndy Chiu
riscv_v_vstate_ctrl_set_current(unsigned long arg)2441fd96a3eSAndy Chiu long riscv_v_vstate_ctrl_set_current(unsigned long arg)
2451fd96a3eSAndy Chiu {
2461fd96a3eSAndy Chiu bool inherit;
2471fd96a3eSAndy Chiu int cur, next;
2481fd96a3eSAndy Chiu
2491fd96a3eSAndy Chiu if (!has_vector())
2501fd96a3eSAndy Chiu return -EINVAL;
2511fd96a3eSAndy Chiu
2521fd96a3eSAndy Chiu if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
2531fd96a3eSAndy Chiu return -EINVAL;
2541fd96a3eSAndy Chiu
2551fd96a3eSAndy Chiu cur = VSTATE_CTRL_GET_CUR(arg);
2561fd96a3eSAndy Chiu switch (cur) {
2571fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_OFF:
2581fd96a3eSAndy Chiu /* Do not allow user to turn off V if current is not off */
2591fd96a3eSAndy Chiu if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
2601fd96a3eSAndy Chiu return -EPERM;
2611fd96a3eSAndy Chiu
2621fd96a3eSAndy Chiu break;
2631fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_ON:
2641fd96a3eSAndy Chiu break;
2651fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
2661fd96a3eSAndy Chiu cur = riscv_v_ctrl_get_cur(current);
2671fd96a3eSAndy Chiu break;
2681fd96a3eSAndy Chiu default:
2691fd96a3eSAndy Chiu return -EINVAL;
2701fd96a3eSAndy Chiu }
2711fd96a3eSAndy Chiu
2721fd96a3eSAndy Chiu next = VSTATE_CTRL_GET_NEXT(arg);
2731fd96a3eSAndy Chiu inherit = VSTATE_CTRL_GET_INHERIT(arg);
2741fd96a3eSAndy Chiu switch (next) {
2751fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
2761fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_OFF:
2771fd96a3eSAndy Chiu case PR_RISCV_V_VSTATE_CTRL_ON:
2781fd96a3eSAndy Chiu riscv_v_ctrl_set(current, cur, next, inherit);
2791fd96a3eSAndy Chiu return 0;
2801fd96a3eSAndy Chiu }
2811fd96a3eSAndy Chiu
2821fd96a3eSAndy Chiu return -EINVAL;
2831fd96a3eSAndy Chiu }
2847ca7a7b9SAndy Chiu
2857ca7a7b9SAndy Chiu #ifdef CONFIG_SYSCTL
2867ca7a7b9SAndy Chiu
2877ca7a7b9SAndy Chiu static struct ctl_table riscv_v_default_vstate_table[] = {
2887ca7a7b9SAndy Chiu {
2897ca7a7b9SAndy Chiu .procname = "riscv_v_default_allow",
2907ca7a7b9SAndy Chiu .data = &riscv_v_implicit_uacc,
2917ca7a7b9SAndy Chiu .maxlen = sizeof(riscv_v_implicit_uacc),
2927ca7a7b9SAndy Chiu .mode = 0644,
2937ca7a7b9SAndy Chiu .proc_handler = proc_dobool,
2947ca7a7b9SAndy Chiu },
2957ca7a7b9SAndy Chiu };
2967ca7a7b9SAndy Chiu
riscv_v_sysctl_init(void)2977ca7a7b9SAndy Chiu static int __init riscv_v_sysctl_init(void)
2987ca7a7b9SAndy Chiu {
2997ca7a7b9SAndy Chiu if (has_vector())
3007ca7a7b9SAndy Chiu if (!register_sysctl("abi", riscv_v_default_vstate_table))
3017ca7a7b9SAndy Chiu return -EINVAL;
3027ca7a7b9SAndy Chiu return 0;
3037ca7a7b9SAndy Chiu }
3047ca7a7b9SAndy Chiu
3057ca7a7b9SAndy Chiu #else /* ! CONFIG_SYSCTL */
riscv_v_sysctl_init(void)3067ca7a7b9SAndy Chiu static int __init riscv_v_sysctl_init(void) { return 0; }
3077ca7a7b9SAndy Chiu #endif /* ! CONFIG_SYSCTL */
3087ca7a7b9SAndy Chiu
riscv_v_init(void)3097ca7a7b9SAndy Chiu static int riscv_v_init(void)
3107ca7a7b9SAndy Chiu {
3117ca7a7b9SAndy Chiu return riscv_v_sysctl_init();
3127ca7a7b9SAndy Chiu }
3137ca7a7b9SAndy Chiu core_initcall(riscv_v_init);
314