1 /* 2 * Copyright (c) 2020 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 #include "opt_cpu.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/sysctl.h> 40 #include <sys/malloc.h> 41 #include <sys/priv.h> 42 #include <sys/flame_graph.h> 43 44 #include <sys/thread2.h> 45 46 #include <machine/atomic.h> 47 #include <machine/cpufunc.h> 48 #include <machine/cputypes.h> 49 #include <machine/psl.h> 50 #include <machine/segments.h> 51 #include <machine/tss.h> 52 #include <machine/specialreg.h> 53 #include <machine/globaldata.h> 54 55 #include <machine/md_var.h> /* setidt() */ 56 57 __read_frequently struct flame_graph_pcpu *flame_graph_array; 58 __read_frequently static int flame_graph_enable; 59 60 SYSCTL_INT(_debug, OID_AUTO, flame_graph_enable, CTLFLAG_RW, 61 &flame_graph_enable, 0, "Collect data for flame graphs"); 62 SYSCTL_LONG(_debug, OID_AUTO, flame_graph_array, CTLFLAG_RD, 63 &flame_graph_array, 0, "Collect data for flame graphs"); 64 65 MALLOC_DEFINE(M_FLAME, "flame_graphs", "Flame Graphs"); 66 67 static void 68 hard_sniff_init(void *arg) 69 { 70 struct flame_graph_pcpu *fg; 71 struct flame_graph_entry *fge; 72 int n; 73 74 flame_graph_array = kmalloc(sizeof(*fg) * ncpus, 75 M_FLAME, M_WAITOK | M_CACHEALIGN | M_ZERO); 76 for (n = 0; n < ncpus; ++n) { 77 fge = kmalloc(sizeof(*fge) * FLAME_GRAPH_NENTRIES, 78 M_FLAME, M_WAITOK | M_CACHEALIGN | M_ZERO); 79 80 fg = &flame_graph_array[n]; 81 fg->nentries = FLAME_GRAPH_NENTRIES; 82 fg->fge = fge; 83 } 84 } 85 86 SYSINIT(swi_vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY, hard_sniff_init, NULL); 87 88 /* 89 * Xsniff vector calls into here 90 * 91 * WARNING! This code ignores critical sections! The system can be 92 * in any state. The only thing we safely have access to 93 * is the gd. 94 */ 95 void 96 hard_sniff(struct trapframe *tf) 97 { 98 globaldata_t gd = mycpu; 99 thread_t td; 100 char *top; 101 char *bot; 102 char *rbp; 103 char *rip; 104 struct flame_graph_pcpu *fg; 105 struct flame_graph_entry *fge; 106 int n; 107 108 /* 109 * systat -pv 1 sampling 110 */ 111 gd->gd_sample_pc = (void *)(intptr_t)tf->tf_rip; 112 gd->gd_sample_sp = (void *)(intptr_t)tf->tf_rsp; 113 114 /* 115 * Flame graph sampling, require %rbp (frame pointer) chaining. 116 * Attempt to follow the chain and record what we believe are 117 * %rip addresses. 118 */ 119 if (flame_graph_enable == 0) 120 return; 121 td = gd->gd_curthread; 122 if (td == NULL) 123 return; 124 bot = (char *)td->td_kstack + PAGE_SIZE; /* skip guard */ 125 top = (char *)td->td_kstack + td->td_kstack_size; 126 if (bot >= top) 127 return; 128 fg = &flame_graph_array[gd->gd_cpuid]; 129 fge = &fg->fge[fg->windex % FLAME_GRAPH_NENTRIES]; 130 131 rip = (char *)(intptr_t)tf->tf_rip; 132 fge->rips[0] = (intptr_t)rip; 133 rbp = (char *)(intptr_t)tf->tf_rbp; 134 135 for (n = 1; n < FLAME_GRAPH_FRAMES - 1; ++n) { 136 if (rbp < bot || rbp > top - 8 || ((intptr_t)rbp & 7)) 137 break; 138 fge->rips[n] = (intptr_t)*(char **)(rbp + 8); 139 if (*(char **)rbp <= rbp) 140 break; 141 rbp = *(char **)rbp; 142 } 143 fge->rips[n] = 0; 144 cpu_sfence(); 145 ++fg->windex; 146 } 147 148 static int 149 sysctl_flame_graph_data(SYSCTL_HANDLER_ARGS) 150 { 151 int error; 152 int n; 153 size_t ebytes; 154 155 error = priv_check_cred(curthread->td_ucred, PRIV_ROOT, 0); 156 if (error) 157 return error; 158 if (flame_graph_array == NULL) 159 return EOPNOTSUPP; 160 161 ebytes = sizeof(struct flame_graph_pcpu) + 162 sizeof(struct flame_graph_entry); 163 164 for (n = 0; n < ncpus && error == 0; ++n) { 165 error = SYSCTL_OUT(req, &ebytes, sizeof(ebytes)); 166 if (error == 0) 167 error = SYSCTL_OUT(req, flame_graph_array + n, 168 sizeof(*flame_graph_array)); 169 if (error == 0) 170 error = SYSCTL_OUT(req, flame_graph_array[n].fge, 171 sizeof(*flame_graph_array->fge) * 172 FLAME_GRAPH_NENTRIES); 173 } 174 175 return error; 176 } 177 178 SYSCTL_PROC(_debug, OID_AUTO, flame_graph_data, 179 (CTLTYPE_OPAQUE|CTLFLAG_RD), 0, 0, 180 sysctl_flame_graph_data, "S,flames", "Flame Graph Data"); 181 182 static int 183 sysctl_flame_graph_sniff(SYSCTL_HANDLER_ARGS) 184 { 185 int error; 186 187 error = priv_check_cred(curthread->td_ucred, PRIV_ROOT, 0); 188 if (error) 189 return error; 190 if (flame_graph_enable == 0) 191 return EINVAL; 192 if (req->newptr) 193 smp_sniff(); 194 return(SYSCTL_OUT(req, &error, sizeof(int))); 195 } 196 197 SYSCTL_PROC(_debug, OID_AUTO, flame_graph_sniff, 198 (CTLTYPE_UINT|CTLFLAG_RW), 0, 0, 199 sysctl_flame_graph_sniff, "IU", "Flame Graph Poll"); 200