1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 5 * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the name of the Politecnico di Torino nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef _BPF_JIT_MACHDEP_H_ 35 #define _BPF_JIT_MACHDEP_H_ 36 37 /* 38 * Registers 39 */ 40 #define RAX 0 41 #define RCX 1 42 #define RDX 2 43 #define RBX 3 44 #define RSP 4 45 #define RBP 5 46 #define RSI 6 47 #define RDI 7 48 #define R8 0 49 #define R9 1 50 #define R10 2 51 #define R11 3 52 #define R12 4 53 #define R13 5 54 #define R14 6 55 #define R15 7 56 57 #define EAX 0 58 #define ECX 1 59 #define EDX 2 60 #define EBX 3 61 #define ESP 4 62 #define EBP 5 63 #define ESI 6 64 #define EDI 7 65 #define R8D 0 66 #define R9D 1 67 #define R10D 2 68 #define R11D 3 69 #define R12D 4 70 #define R13D 5 71 #define R14D 6 72 #define R15D 7 73 74 #define AX 0 75 #define CX 1 76 #define DX 2 77 #define BX 3 78 #define SP 4 79 #define BP 5 80 #define SI 6 81 #define DI 7 82 83 #define AL 0 84 #define CL 1 85 #define DL 2 86 #define BL 3 87 88 /* Optimization flags */ 89 #define BPF_JIT_FRET 0x01 90 #define BPF_JIT_FPKT 0x02 91 #define BPF_JIT_FMEM 0x04 92 #define BPF_JIT_FJMP 0x08 93 #define BPF_JIT_FLEN 0x10 94 95 #define BPF_JIT_FLAG_ALL \ 96 (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN) 97 98 /* A stream of native binary code */ 99 typedef struct bpf_bin_stream { 100 /* Current native instruction pointer. */ 101 int cur_ip; 102 103 /* 104 * Current BPF instruction pointer, i.e. position in 105 * the BPF program reached by the jitter. 106 */ 107 int bpf_pc; 108 109 /* Instruction buffer, contains the generated native code. */ 110 char *ibuf; 111 112 /* Jumps reference table. */ 113 u_int *refs; 114 } bpf_bin_stream; 115 116 /* 117 * Prototype of the emit functions. 118 * 119 * Different emit functions are used to create the reference table and 120 * to generate the actual filtering code. This allows to have simpler 121 * instruction macros. 122 * The first parameter is the stream that will receive the data. 123 * The second one is a variable containing the data. 124 * The third one is the length, that can be 1, 2, or 4 since it is possible 125 * to emit a byte, a short, or a word at a time. 126 */ 127 typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); 128 129 /* 130 * Native instruction macros 131 */ 132 133 /* movl i32,r32 */ 134 #define MOVid(i32, r32) do { \ 135 emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1); \ 136 emitm(&stream, i32, 4); \ 137 } while (0) 138 139 /* movq i64,r64 */ 140 #define MOViq(i64, r64) do { \ 141 emitm(&stream, 0x48, 1); \ 142 emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1); \ 143 emitm(&stream, i64, 4); \ 144 emitm(&stream, (i64 >> 32), 4); \ 145 } while (0) 146 147 /* movl sr32,dr32 */ 148 #define MOVrd(sr32, dr32) do { \ 149 emitm(&stream, 0x89, 1); \ 150 emitm(&stream, \ 151 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 152 } while (0) 153 154 /* movl sr32,dr32 (dr32 = %r8-15d) */ 155 #define MOVrd2(sr32, dr32) do { \ 156 emitm(&stream, 0x8941, 2); \ 157 emitm(&stream, \ 158 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 159 } while (0) 160 161 /* movl sr32,dr32 (sr32 = %r8-15d) */ 162 #define MOVrd3(sr32, dr32) do { \ 163 emitm(&stream, 0x8944, 2); \ 164 emitm(&stream, \ 165 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 166 } while (0) 167 168 /* movq sr64,dr64 */ 169 #define MOVrq(sr64, dr64) do { \ 170 emitm(&stream, 0x8948, 2); \ 171 emitm(&stream, \ 172 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 173 } while (0) 174 175 /* movq sr64,dr64 (dr64 = %r8-15) */ 176 #define MOVrq2(sr64, dr64) do { \ 177 emitm(&stream, 0x8949, 2); \ 178 emitm(&stream, \ 179 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 180 } while (0) 181 182 /* movq sr64,dr64 (sr64 = %r8-15) */ 183 #define MOVrq3(sr64, dr64) do { \ 184 emitm(&stream, 0x894c, 2); \ 185 emitm(&stream, \ 186 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 187 } while (0) 188 189 /* movl (sr64,or64,1),dr32 */ 190 #define MOVobd(sr64, or64, dr32) do { \ 191 emitm(&stream, 0x8b, 1); \ 192 emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1); \ 193 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 194 } while (0) 195 196 /* movw (sr64,or64,1),dr16 */ 197 #define MOVobw(sr64, or64, dr16) do { \ 198 emitm(&stream, 0x8b66, 2); \ 199 emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1); \ 200 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 201 } while (0) 202 203 /* movb (sr64,or64,1),dr8 */ 204 #define MOVobb(sr64, or64, dr8) do { \ 205 emitm(&stream, 0x8a, 1); \ 206 emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1); \ 207 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 208 } while (0) 209 210 /* movl sr32,(dr64,or64,1) */ 211 #define MOVomd(sr32, dr64, or64) do { \ 212 emitm(&stream, 0x89, 1); \ 213 emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1); \ 214 emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 215 } while (0) 216 217 /* bswapl dr32 */ 218 #define BSWAP(dr32) do { \ 219 emitm(&stream, 0xf, 1); \ 220 emitm(&stream, (0x19 << 3) | dr32, 1); \ 221 } while (0) 222 223 /* xchgb %al,%ah */ 224 #define SWAP_AX() do { \ 225 emitm(&stream, 0xc486, 2); \ 226 } while (0) 227 228 /* pushq r64 */ 229 #define PUSH(r64) do { \ 230 emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \ 231 } while (0) 232 233 /* leaveq */ 234 #define LEAVE() do { \ 235 emitm(&stream, 0xc9, 1); \ 236 } while (0) 237 238 /* retq */ 239 #define RET() do { \ 240 emitm(&stream, 0xc3, 1); \ 241 } while (0) 242 243 /* addl sr32,dr32 */ 244 #define ADDrd(sr32, dr32) do { \ 245 emitm(&stream, 0x01, 1); \ 246 emitm(&stream, \ 247 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 248 } while (0) 249 250 /* addl i32,%eax */ 251 #define ADD_EAXi(i32) do { \ 252 emitm(&stream, 0x05, 1); \ 253 emitm(&stream, i32, 4); \ 254 } while (0) 255 256 /* addl i8,r32 */ 257 #define ADDib(i8, r32) do { \ 258 emitm(&stream, 0x83, 1); \ 259 emitm(&stream, (24 << 3) | r32, 1); \ 260 emitm(&stream, i8, 1); \ 261 } while (0) 262 263 /* subl sr32,dr32 */ 264 #define SUBrd(sr32, dr32) do { \ 265 emitm(&stream, 0x29, 1); \ 266 emitm(&stream, \ 267 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 268 } while (0) 269 270 /* subl i32,%eax */ 271 #define SUB_EAXi(i32) do { \ 272 emitm(&stream, 0x2d, 1); \ 273 emitm(&stream, i32, 4); \ 274 } while (0) 275 276 /* subq i8,r64 */ 277 #define SUBib(i8, r64) do { \ 278 emitm(&stream, 0x8348, 2); \ 279 emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \ 280 emitm(&stream, i8, 1); \ 281 } while (0) 282 283 /* mull r32 */ 284 #define MULrd(r32) do { \ 285 emitm(&stream, 0xf7, 1); \ 286 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 287 } while (0) 288 289 /* divl r32 */ 290 #define DIVrd(r32) do { \ 291 emitm(&stream, 0xf7, 1); \ 292 emitm(&stream, (15 << 4) | (r32 & 0x7), 1); \ 293 } while (0) 294 295 /* andb i8,r8 */ 296 #define ANDib(i8, r8) do { \ 297 if (r8 == AL) { \ 298 emitm(&stream, 0x24, 1); \ 299 } else { \ 300 emitm(&stream, 0x80, 1); \ 301 emitm(&stream, (7 << 5) | r8, 1); \ 302 } \ 303 emitm(&stream, i8, 1); \ 304 } while (0) 305 306 /* andl i32,r32 */ 307 #define ANDid(i32, r32) do { \ 308 if (r32 == EAX) { \ 309 emitm(&stream, 0x25, 1); \ 310 } else { \ 311 emitm(&stream, 0x81, 1); \ 312 emitm(&stream, (7 << 5) | r32, 1); \ 313 } \ 314 emitm(&stream, i32, 4); \ 315 } while (0) 316 317 /* andl sr32,dr32 */ 318 #define ANDrd(sr32, dr32) do { \ 319 emitm(&stream, 0x21, 1); \ 320 emitm(&stream, \ 321 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 322 } while (0) 323 324 /* testl i32,r32 */ 325 #define TESTid(i32, r32) do { \ 326 if (r32 == EAX) { \ 327 emitm(&stream, 0xa9, 1); \ 328 } else { \ 329 emitm(&stream, 0xf7, 1); \ 330 emitm(&stream, (3 << 6) | r32, 1); \ 331 } \ 332 emitm(&stream, i32, 4); \ 333 } while (0) 334 335 /* testl sr32,dr32 */ 336 #define TESTrd(sr32, dr32) do { \ 337 emitm(&stream, 0x85, 1); \ 338 emitm(&stream, \ 339 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 340 } while (0) 341 342 /* orl sr32,dr32 */ 343 #define ORrd(sr32, dr32) do { \ 344 emitm(&stream, 0x09, 1); \ 345 emitm(&stream, \ 346 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 347 } while (0) 348 349 /* orl i32,r32 */ 350 #define ORid(i32, r32) do { \ 351 if (r32 == EAX) { \ 352 emitm(&stream, 0x0d, 1); \ 353 } else { \ 354 emitm(&stream, 0x81, 1); \ 355 emitm(&stream, (25 << 3) | r32, 1); \ 356 } \ 357 emitm(&stream, i32, 4); \ 358 } while (0) 359 360 /* xorl sr32,dr32 */ 361 #define XORrd(sr32, dr32) do { \ 362 emitm(&stream, 0x31, 1); \ 363 emitm(&stream, \ 364 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 365 } while (0) 366 367 /* xorl i32,r32 */ 368 #define XORid(i32, r32) do { \ 369 if (r32 == EAX) { \ 370 emitm(&stream, 0x35, 1); \ 371 } else { \ 372 emitm(&stream, 0x81, 1); \ 373 emitm(&stream, (25 << 3) | r32, 1); \ 374 } \ 375 emitm(&stream, i32, 4); \ 376 } while (0) 377 378 /* shll i8,r32 */ 379 #define SHLib(i8, r32) do { \ 380 emitm(&stream, 0xc1, 1); \ 381 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 382 emitm(&stream, i8, 1); \ 383 } while (0) 384 385 /* shll %cl,dr32 */ 386 #define SHL_CLrb(dr32) do { \ 387 emitm(&stream, 0xd3, 1); \ 388 emitm(&stream, (7 << 5) | (dr32 & 0x7), 1); \ 389 } while (0) 390 391 /* shrl i8,r32 */ 392 #define SHRib(i8, r32) do { \ 393 emitm(&stream, 0xc1, 1); \ 394 emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \ 395 emitm(&stream, i8, 1); \ 396 } while (0) 397 398 /* shrl %cl,dr32 */ 399 #define SHR_CLrb(dr32) do { \ 400 emitm(&stream, 0xd3, 1); \ 401 emitm(&stream, (29 << 3) | (dr32 & 0x7), 1); \ 402 } while (0) 403 404 /* negl r32 */ 405 #define NEGd(r32) do { \ 406 emitm(&stream, 0xf7, 1); \ 407 emitm(&stream, (27 << 3) | (r32 & 0x7), 1); \ 408 } while (0) 409 410 /* cmpl sr32,dr32 */ 411 #define CMPrd(sr32, dr32) do { \ 412 emitm(&stream, 0x39, 1); \ 413 emitm(&stream, \ 414 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 415 } while (0) 416 417 /* cmpl i32,dr32 */ 418 #define CMPid(i32, dr32) do { \ 419 if (dr32 == EAX){ \ 420 emitm(&stream, 0x3d, 1); \ 421 emitm(&stream, i32, 4); \ 422 } else { \ 423 emitm(&stream, 0x81, 1); \ 424 emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1); \ 425 emitm(&stream, i32, 4); \ 426 } \ 427 } while (0) 428 429 /* jb off8 */ 430 #define JBb(off8) do { \ 431 emitm(&stream, 0x72, 1); \ 432 emitm(&stream, off8, 1); \ 433 } while (0) 434 435 /* jae off8 */ 436 #define JAEb(off8) do { \ 437 emitm(&stream, 0x73, 1); \ 438 emitm(&stream, off8, 1); \ 439 } while (0) 440 441 /* jne off8 */ 442 #define JNEb(off8) do { \ 443 emitm(&stream, 0x75, 1); \ 444 emitm(&stream, off8, 1); \ 445 } while (0) 446 447 /* ja off8 */ 448 #define JAb(off8) do { \ 449 emitm(&stream, 0x77, 1); \ 450 emitm(&stream, off8, 1); \ 451 } while (0) 452 453 /* jmp off32 */ 454 #define JMP(off32) do { \ 455 emitm(&stream, 0xe9, 1); \ 456 emitm(&stream, off32, 4); \ 457 } while (0) 458 459 /* xorl r32,r32 */ 460 #define ZEROrd(r32) do { \ 461 emitm(&stream, 0x31, 1); \ 462 emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \ 463 } while (0) 464 465 /* 466 * Conditional long jumps 467 */ 468 #define JB 0x82 469 #define JAE 0x83 470 #define JE 0x84 471 #define JNE 0x85 472 #define JBE 0x86 473 #define JA 0x87 474 475 #define JCC(t, f) do { \ 476 if (ins->jt != 0 && ins->jf != 0) { \ 477 /* 5 is the size of the following jmp */ \ 478 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 479 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 480 stream.refs[stream.bpf_pc] + 5, 4); \ 481 JMP(stream.refs[stream.bpf_pc + ins->jf] - \ 482 stream.refs[stream.bpf_pc]); \ 483 } else if (ins->jt != 0) { \ 484 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 485 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 486 stream.refs[stream.bpf_pc], 4); \ 487 } else { \ 488 emitm(&stream, ((f) << 8) | 0x0f, 2); \ 489 emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] - \ 490 stream.refs[stream.bpf_pc], 4); \ 491 } \ 492 } while (0) 493 494 #define JUMP(off) do { \ 495 if ((off) != 0) \ 496 JMP(stream.refs[stream.bpf_pc + (off)] - \ 497 stream.refs[stream.bpf_pc]); \ 498 } while (0) 499 500 #endif /* _BPF_JIT_MACHDEP_H_ */ 501