1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kernel.h> 26 #include <asm/fpu/api.h> 27 28 #include "i915_drv.h" 29 30 #ifdef notyet 31 static DEFINE_STATIC_KEY_FALSE(has_movntdqa); 32 #endif 33 34 #ifdef CONFIG_AS_MOVNTDQA 35 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) 36 { 37 kernel_fpu_begin(); 38 39 len >>= 4; 40 while (len >= 4) { 41 asm("movntdqa (%0), %%xmm0\n" 42 "movntdqa 16(%0), %%xmm1\n" 43 "movntdqa 32(%0), %%xmm2\n" 44 "movntdqa 48(%0), %%xmm3\n" 45 "movaps %%xmm0, (%1)\n" 46 "movaps %%xmm1, 16(%1)\n" 47 "movaps %%xmm2, 32(%1)\n" 48 "movaps %%xmm3, 48(%1)\n" 49 :: "r" (src), "r" (dst) : "memory"); 50 src += 64; 51 dst += 64; 52 len -= 4; 53 } 54 while (len--) { 55 asm("movntdqa (%0), %%xmm0\n" 56 "movaps %%xmm0, (%1)\n" 57 :: "r" (src), "r" (dst) : "memory"); 58 src += 16; 59 dst += 16; 60 } 61 62 kernel_fpu_end(); 63 } 64 #endif 65 66 /** 67 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC 68 * @dst: destination pointer 69 * @src: source pointer 70 * @len: how many bytes to copy 71 * 72 * i915_memcpy_from_wc copies @len bytes from @src to @dst using 73 * non-temporal instructions where available. Note that all arguments 74 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple 75 * of 16. 76 * 77 * To test whether accelerated reads from WC are supported, use 78 * i915_memcpy_from_wc(NULL, NULL, 0); 79 * 80 * Returns true if the copy was successful, false if the preconditions 81 * are not met. 82 */ 83 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) 84 { 85 if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15)) 86 return false; 87 88 #ifdef CONFIG_AS_MOVNTDQA 89 if (static_branch_likely(&has_movntdqa)) { 90 if (likely(len)) 91 __memcpy_ntdqa(dst, src, len); 92 return true; 93 } 94 #endif 95 96 return false; 97 } 98 99 void i915_memcpy_init_early(struct drm_i915_private *dev_priv) 100 { 101 #ifdef notyet 102 /* 103 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions 104 * emulation. So don't enable movntdqa in hypervisor guest. 105 */ 106 if (static_cpu_has(X86_FEATURE_XMM4_1) && 107 !boot_cpu_has(X86_FEATURE_HYPERVISOR)) 108 static_branch_enable(&has_movntdqa); 109 #endif 110 } 111