1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 5 * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the name of the Politecnico di Torino nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36 #ifndef _BPF_JIT_MACHDEP_H_ 37 #define _BPF_JIT_MACHDEP_H_ 38 39 /* 40 * Registers 41 */ 42 #define RAX 0 43 #define RCX 1 44 #define RDX 2 45 #define RBX 3 46 #define RSP 4 47 #define RBP 5 48 #define RSI 6 49 #define RDI 7 50 #define R8 0 51 #define R9 1 52 #define R10 2 53 #define R11 3 54 #define R12 4 55 #define R13 5 56 #define R14 6 57 #define R15 7 58 59 #define EAX 0 60 #define ECX 1 61 #define EDX 2 62 #define EBX 3 63 #define ESP 4 64 #define EBP 5 65 #define ESI 6 66 #define EDI 7 67 #define R8D 0 68 #define R9D 1 69 #define R10D 2 70 #define R11D 3 71 #define R12D 4 72 #define R13D 5 73 #define R14D 6 74 #define R15D 7 75 76 #define AX 0 77 #define CX 1 78 #define DX 2 79 #define BX 3 80 #define SP 4 81 #define BP 5 82 #define SI 6 83 #define DI 7 84 85 #define AL 0 86 #define CL 1 87 #define DL 2 88 #define BL 3 89 90 /* Optimization flags */ 91 #define BPF_JIT_FRET 0x01 92 #define BPF_JIT_FPKT 0x02 93 #define BPF_JIT_FMEM 0x04 94 #define BPF_JIT_FJMP 0x08 95 #define BPF_JIT_FLEN 0x10 96 97 #define BPF_JIT_FLAG_ALL \ 98 (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN) 99 100 /* A stream of native binary code */ 101 typedef struct bpf_bin_stream { 102 /* Current native instruction pointer. */ 103 int cur_ip; 104 105 /* 106 * Current BPF instruction pointer, i.e. position in 107 * the BPF program reached by the jitter. 108 */ 109 int bpf_pc; 110 111 /* Instruction buffer, contains the generated native code. */ 112 char *ibuf; 113 114 /* Jumps reference table. */ 115 u_int *refs; 116 } bpf_bin_stream; 117 118 /* 119 * Prototype of the emit functions. 120 * 121 * Different emit functions are used to create the reference table and 122 * to generate the actual filtering code. This allows to have simpler 123 * instruction macros. 124 * The first parameter is the stream that will receive the data. 125 * The second one is a variable containing the data. 126 * The third one is the length, that can be 1, 2, or 4 since it is possible 127 * to emit a byte, a short, or a word at a time. 128 */ 129 typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); 130 131 /* 132 * Native instruction macros 133 */ 134 135 /* movl i32,r32 */ 136 #define MOVid(i32, r32) do { \ 137 emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1); \ 138 emitm(&stream, i32, 4); \ 139 } while (0) 140 141 /* movq i64,r64 */ 142 #define MOViq(i64, r64) do { \ 143 emitm(&stream, 0x48, 1); \ 144 emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1); \ 145 emitm(&stream, i64, 4); \ 146 emitm(&stream, (i64 >> 32), 4); \ 147 } while (0) 148 149 /* movl sr32,dr32 */ 150 #define MOVrd(sr32, dr32) do { \ 151 emitm(&stream, 0x89, 1); \ 152 emitm(&stream, \ 153 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 154 } while (0) 155 156 /* movl sr32,dr32 (dr32 = %r8-15d) */ 157 #define MOVrd2(sr32, dr32) do { \ 158 emitm(&stream, 0x8941, 2); \ 159 emitm(&stream, \ 160 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 161 } while (0) 162 163 /* movl sr32,dr32 (sr32 = %r8-15d) */ 164 #define MOVrd3(sr32, dr32) do { \ 165 emitm(&stream, 0x8944, 2); \ 166 emitm(&stream, \ 167 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 168 } while (0) 169 170 /* movq sr64,dr64 */ 171 #define MOVrq(sr64, dr64) do { \ 172 emitm(&stream, 0x8948, 2); \ 173 emitm(&stream, \ 174 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 175 } while (0) 176 177 /* movq sr64,dr64 (dr64 = %r8-15) */ 178 #define MOVrq2(sr64, dr64) do { \ 179 emitm(&stream, 0x8949, 2); \ 180 emitm(&stream, \ 181 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 182 } while (0) 183 184 /* movq sr64,dr64 (sr64 = %r8-15) */ 185 #define MOVrq3(sr64, dr64) do { \ 186 emitm(&stream, 0x894c, 2); \ 187 emitm(&stream, \ 188 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 189 } while (0) 190 191 /* movl (sr64,or64,1),dr32 */ 192 #define MOVobd(sr64, or64, dr32) do { \ 193 emitm(&stream, 0x8b, 1); \ 194 emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1); \ 195 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 196 } while (0) 197 198 /* movw (sr64,or64,1),dr16 */ 199 #define MOVobw(sr64, or64, dr16) do { \ 200 emitm(&stream, 0x8b66, 2); \ 201 emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1); \ 202 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 203 } while (0) 204 205 /* movb (sr64,or64,1),dr8 */ 206 #define MOVobb(sr64, or64, dr8) do { \ 207 emitm(&stream, 0x8a, 1); \ 208 emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1); \ 209 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 210 } while (0) 211 212 /* movl sr32,(dr64,or64,1) */ 213 #define MOVomd(sr32, dr64, or64) do { \ 214 emitm(&stream, 0x89, 1); \ 215 emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1); \ 216 emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 217 } while (0) 218 219 /* bswapl dr32 */ 220 #define BSWAP(dr32) do { \ 221 emitm(&stream, 0xf, 1); \ 222 emitm(&stream, (0x19 << 3) | dr32, 1); \ 223 } while (0) 224 225 /* xchgb %al,%ah */ 226 #define SWAP_AX() do { \ 227 emitm(&stream, 0xc486, 2); \ 228 } while (0) 229 230 /* pushq r64 */ 231 #define PUSH(r64) do { \ 232 emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \ 233 } while (0) 234 235 /* leaveq */ 236 #define LEAVE() do { \ 237 emitm(&stream, 0xc9, 1); \ 238 } while (0) 239 240 /* retq */ 241 #define RET() do { \ 242 emitm(&stream, 0xc3, 1); \ 243 } while (0) 244 245 /* addl sr32,dr32 */ 246 #define ADDrd(sr32, dr32) do { \ 247 emitm(&stream, 0x01, 1); \ 248 emitm(&stream, \ 249 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 250 } while (0) 251 252 /* addl i32,%eax */ 253 #define ADD_EAXi(i32) do { \ 254 emitm(&stream, 0x05, 1); \ 255 emitm(&stream, i32, 4); \ 256 } while (0) 257 258 /* addl i8,r32 */ 259 #define ADDib(i8, r32) do { \ 260 emitm(&stream, 0x83, 1); \ 261 emitm(&stream, (24 << 3) | r32, 1); \ 262 emitm(&stream, i8, 1); \ 263 } while (0) 264 265 /* subl sr32,dr32 */ 266 #define SUBrd(sr32, dr32) do { \ 267 emitm(&stream, 0x29, 1); \ 268 emitm(&stream, \ 269 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 270 } while (0) 271 272 /* subl i32,%eax */ 273 #define SUB_EAXi(i32) do { \ 274 emitm(&stream, 0x2d, 1); \ 275 emitm(&stream, i32, 4); \ 276 } while (0) 277 278 /* subq i8,r64 */ 279 #define SUBib(i8, r64) do { \ 280 emitm(&stream, 0x8348, 2); \ 281 emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \ 282 emitm(&stream, i8, 1); \ 283 } while (0) 284 285 /* mull r32 */ 286 #define MULrd(r32) do { \ 287 emitm(&stream, 0xf7, 1); \ 288 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 289 } while (0) 290 291 /* divl r32 */ 292 #define DIVrd(r32) do { \ 293 emitm(&stream, 0xf7, 1); \ 294 emitm(&stream, (15 << 4) | (r32 & 0x7), 1); \ 295 } while (0) 296 297 /* andb i8,r8 */ 298 #define ANDib(i8, r8) do { \ 299 if (r8 == AL) { \ 300 emitm(&stream, 0x24, 1); \ 301 } else { \ 302 emitm(&stream, 0x80, 1); \ 303 emitm(&stream, (7 << 5) | r8, 1); \ 304 } \ 305 emitm(&stream, i8, 1); \ 306 } while (0) 307 308 /* andl i32,r32 */ 309 #define ANDid(i32, r32) do { \ 310 if (r32 == EAX) { \ 311 emitm(&stream, 0x25, 1); \ 312 } else { \ 313 emitm(&stream, 0x81, 1); \ 314 emitm(&stream, (7 << 5) | r32, 1); \ 315 } \ 316 emitm(&stream, i32, 4); \ 317 } while (0) 318 319 /* andl sr32,dr32 */ 320 #define ANDrd(sr32, dr32) do { \ 321 emitm(&stream, 0x21, 1); \ 322 emitm(&stream, \ 323 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 324 } while (0) 325 326 /* testl i32,r32 */ 327 #define TESTid(i32, r32) do { \ 328 if (r32 == EAX) { \ 329 emitm(&stream, 0xa9, 1); \ 330 } else { \ 331 emitm(&stream, 0xf7, 1); \ 332 emitm(&stream, (3 << 6) | r32, 1); \ 333 } \ 334 emitm(&stream, i32, 4); \ 335 } while (0) 336 337 /* testl sr32,dr32 */ 338 #define TESTrd(sr32, dr32) do { \ 339 emitm(&stream, 0x85, 1); \ 340 emitm(&stream, \ 341 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 342 } while (0) 343 344 /* orl sr32,dr32 */ 345 #define ORrd(sr32, dr32) do { \ 346 emitm(&stream, 0x09, 1); \ 347 emitm(&stream, \ 348 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 349 } while (0) 350 351 /* orl i32,r32 */ 352 #define ORid(i32, r32) do { \ 353 if (r32 == EAX) { \ 354 emitm(&stream, 0x0d, 1); \ 355 } else { \ 356 emitm(&stream, 0x81, 1); \ 357 emitm(&stream, (25 << 3) | r32, 1); \ 358 } \ 359 emitm(&stream, i32, 4); \ 360 } while (0) 361 362 /* xorl sr32,dr32 */ 363 #define XORrd(sr32, dr32) do { \ 364 emitm(&stream, 0x31, 1); \ 365 emitm(&stream, \ 366 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 367 } while (0) 368 369 /* xorl i32,r32 */ 370 #define XORid(i32, r32) do { \ 371 if (r32 == EAX) { \ 372 emitm(&stream, 0x35, 1); \ 373 } else { \ 374 emitm(&stream, 0x81, 1); \ 375 emitm(&stream, (25 << 3) | r32, 1); \ 376 } \ 377 emitm(&stream, i32, 4); \ 378 } while (0) 379 380 /* shll i8,r32 */ 381 #define SHLib(i8, r32) do { \ 382 emitm(&stream, 0xc1, 1); \ 383 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 384 emitm(&stream, i8, 1); \ 385 } while (0) 386 387 /* shll %cl,dr32 */ 388 #define SHL_CLrb(dr32) do { \ 389 emitm(&stream, 0xd3, 1); \ 390 emitm(&stream, (7 << 5) | (dr32 & 0x7), 1); \ 391 } while (0) 392 393 /* shrl i8,r32 */ 394 #define SHRib(i8, r32) do { \ 395 emitm(&stream, 0xc1, 1); \ 396 emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \ 397 emitm(&stream, i8, 1); \ 398 } while (0) 399 400 /* shrl %cl,dr32 */ 401 #define SHR_CLrb(dr32) do { \ 402 emitm(&stream, 0xd3, 1); \ 403 emitm(&stream, (29 << 3) | (dr32 & 0x7), 1); \ 404 } while (0) 405 406 /* negl r32 */ 407 #define NEGd(r32) do { \ 408 emitm(&stream, 0xf7, 1); \ 409 emitm(&stream, (27 << 3) | (r32 & 0x7), 1); \ 410 } while (0) 411 412 /* cmpl sr32,dr32 */ 413 #define CMPrd(sr32, dr32) do { \ 414 emitm(&stream, 0x39, 1); \ 415 emitm(&stream, \ 416 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 417 } while (0) 418 419 /* cmpl i32,dr32 */ 420 #define CMPid(i32, dr32) do { \ 421 if (dr32 == EAX){ \ 422 emitm(&stream, 0x3d, 1); \ 423 emitm(&stream, i32, 4); \ 424 } else { \ 425 emitm(&stream, 0x81, 1); \ 426 emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1); \ 427 emitm(&stream, i32, 4); \ 428 } \ 429 } while (0) 430 431 /* jb off8 */ 432 #define JBb(off8) do { \ 433 emitm(&stream, 0x72, 1); \ 434 emitm(&stream, off8, 1); \ 435 } while (0) 436 437 /* jae off8 */ 438 #define JAEb(off8) do { \ 439 emitm(&stream, 0x73, 1); \ 440 emitm(&stream, off8, 1); \ 441 } while (0) 442 443 /* jne off8 */ 444 #define JNEb(off8) do { \ 445 emitm(&stream, 0x75, 1); \ 446 emitm(&stream, off8, 1); \ 447 } while (0) 448 449 /* ja off8 */ 450 #define JAb(off8) do { \ 451 emitm(&stream, 0x77, 1); \ 452 emitm(&stream, off8, 1); \ 453 } while (0) 454 455 /* jmp off32 */ 456 #define JMP(off32) do { \ 457 emitm(&stream, 0xe9, 1); \ 458 emitm(&stream, off32, 4); \ 459 } while (0) 460 461 /* xorl r32,r32 */ 462 #define ZEROrd(r32) do { \ 463 emitm(&stream, 0x31, 1); \ 464 emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \ 465 } while (0) 466 467 /* 468 * Conditional long jumps 469 */ 470 #define JB 0x82 471 #define JAE 0x83 472 #define JE 0x84 473 #define JNE 0x85 474 #define JBE 0x86 475 #define JA 0x87 476 477 #define JCC(t, f) do { \ 478 if (ins->jt != 0 && ins->jf != 0) { \ 479 /* 5 is the size of the following jmp */ \ 480 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 481 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 482 stream.refs[stream.bpf_pc] + 5, 4); \ 483 JMP(stream.refs[stream.bpf_pc + ins->jf] - \ 484 stream.refs[stream.bpf_pc]); \ 485 } else if (ins->jt != 0) { \ 486 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 487 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 488 stream.refs[stream.bpf_pc], 4); \ 489 } else { \ 490 emitm(&stream, ((f) << 8) | 0x0f, 2); \ 491 emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] - \ 492 stream.refs[stream.bpf_pc], 4); \ 493 } \ 494 } while (0) 495 496 #define JUMP(off) do { \ 497 if ((off) != 0) \ 498 JMP(stream.refs[stream.bpf_pc + (off)] - \ 499 stream.refs[stream.bpf_pc]); \ 500 } while (0) 501 502 #endif /* _BPF_JIT_MACHDEP_H_ */ 503