1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 static inline bool rvc_enabled(void) 17 { 18 return IS_ENABLED(CONFIG_RISCV_ISA_C); 19 } 20 21 static inline bool rvzbb_enabled(void) 22 { 23 return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB); 24 } 25 26 enum { 27 RV_REG_ZERO = 0, /* The constant value 0 */ 28 RV_REG_RA = 1, /* Return address */ 29 RV_REG_SP = 2, /* Stack pointer */ 30 RV_REG_GP = 3, /* Global pointer */ 31 RV_REG_TP = 4, /* Thread pointer */ 32 RV_REG_T0 = 5, /* Temporaries */ 33 RV_REG_T1 = 6, 34 RV_REG_T2 = 7, 35 RV_REG_FP = 8, /* Saved register/frame pointer */ 36 RV_REG_S1 = 9, /* Saved register */ 37 RV_REG_A0 = 10, /* Function argument/return values */ 38 RV_REG_A1 = 11, /* Function arguments */ 39 RV_REG_A2 = 12, 40 RV_REG_A3 = 13, 41 RV_REG_A4 = 14, 42 RV_REG_A5 = 15, 43 RV_REG_A6 = 16, 44 RV_REG_A7 = 17, 45 RV_REG_S2 = 18, /* Saved registers */ 46 RV_REG_S3 = 19, 47 RV_REG_S4 = 20, 48 RV_REG_S5 = 21, 49 RV_REG_S6 = 22, 50 RV_REG_S7 = 23, 51 RV_REG_S8 = 24, 52 RV_REG_S9 = 25, 53 RV_REG_S10 = 26, 54 RV_REG_S11 = 27, 55 RV_REG_T3 = 28, /* Temporaries */ 56 RV_REG_T4 = 29, 57 RV_REG_T5 = 30, 58 RV_REG_T6 = 31, 59 }; 60 61 static inline bool is_creg(u8 reg) 62 { 63 return (1 << reg) & (BIT(RV_REG_FP) | 64 BIT(RV_REG_S1) | 65 BIT(RV_REG_A0) | 66 BIT(RV_REG_A1) | 67 BIT(RV_REG_A2) | 68 BIT(RV_REG_A3) | 69 BIT(RV_REG_A4) | 70 BIT(RV_REG_A5)); 71 } 72 73 struct rv_jit_context { 74 struct bpf_prog *prog; 75 u16 *insns; /* RV insns */ 76 u16 *ro_insns; 77 int ninsns; 78 int prologue_len; 79 int epilogue_offset; 80 int *offset; /* BPF to RV */ 81 int nexentries; 82 unsigned long flags; 83 int stack_size; 84 }; 85 86 /* Convert from ninsns to bytes. */ 87 static inline int ninsns_rvoff(int ninsns) 88 { 89 return ninsns << 1; 90 } 91 92 struct rv_jit_data { 93 struct bpf_binary_header *header; 94 struct bpf_binary_header *ro_header; 95 u8 *image; 96 u8 *ro_image; 97 struct rv_jit_context ctx; 98 }; 99 100 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 101 { 102 memset(area, 0, size); 103 } 104 105 static inline void bpf_flush_icache(void *start, void *end) 106 { 107 flush_icache_range((unsigned long)start, (unsigned long)end); 108 } 109 110 /* Emit a 4-byte riscv instruction. */ 111 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 112 { 113 if (ctx->insns) { 114 ctx->insns[ctx->ninsns] = insn; 115 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 116 } 117 118 ctx->ninsns += 2; 119 } 120 121 /* Emit a 2-byte riscv compressed instruction. */ 122 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 123 { 124 BUILD_BUG_ON(!rvc_enabled()); 125 126 if (ctx->insns) 127 ctx->insns[ctx->ninsns] = insn; 128 129 ctx->ninsns++; 130 } 131 132 static inline int epilogue_offset(struct rv_jit_context *ctx) 133 { 134 int to = ctx->epilogue_offset, from = ctx->ninsns; 135 136 return ninsns_rvoff(to - from); 137 } 138 139 /* Return -1 or inverted cond. */ 140 static inline int invert_bpf_cond(u8 cond) 141 { 142 switch (cond) { 143 case BPF_JEQ: 144 return BPF_JNE; 145 case BPF_JGT: 146 return BPF_JLE; 147 case BPF_JLT: 148 return BPF_JGE; 149 case BPF_JGE: 150 return BPF_JLT; 151 case BPF_JLE: 152 return BPF_JGT; 153 case BPF_JNE: 154 return BPF_JEQ; 155 case BPF_JSGT: 156 return BPF_JSLE; 157 case BPF_JSLT: 158 return BPF_JSGE; 159 case BPF_JSGE: 160 return BPF_JSLT; 161 case BPF_JSLE: 162 return BPF_JSGT; 163 } 164 return -1; 165 } 166 167 static inline bool is_6b_int(long val) 168 { 169 return -(1L << 5) <= val && val < (1L << 5); 170 } 171 172 static inline bool is_7b_uint(unsigned long val) 173 { 174 return val < (1UL << 7); 175 } 176 177 static inline bool is_8b_uint(unsigned long val) 178 { 179 return val < (1UL << 8); 180 } 181 182 static inline bool is_9b_uint(unsigned long val) 183 { 184 return val < (1UL << 9); 185 } 186 187 static inline bool is_10b_int(long val) 188 { 189 return -(1L << 9) <= val && val < (1L << 9); 190 } 191 192 static inline bool is_10b_uint(unsigned long val) 193 { 194 return val < (1UL << 10); 195 } 196 197 static inline bool is_12b_int(long val) 198 { 199 return -(1L << 11) <= val && val < (1L << 11); 200 } 201 202 static inline int is_12b_check(int off, int insn) 203 { 204 if (!is_12b_int(off)) { 205 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 206 insn, (int)off); 207 return -1; 208 } 209 return 0; 210 } 211 212 static inline bool is_13b_int(long val) 213 { 214 return -(1L << 12) <= val && val < (1L << 12); 215 } 216 217 static inline bool is_21b_int(long val) 218 { 219 return -(1L << 20) <= val && val < (1L << 20); 220 } 221 222 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 223 { 224 int from, to; 225 226 off++; /* BPF branch is from PC+1, RV is from PC */ 227 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; 228 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; 229 return ninsns_rvoff(to - from); 230 } 231 232 /* Instruction formats. */ 233 234 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 235 u8 opcode) 236 { 237 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 238 (rd << 7) | opcode; 239 } 240 241 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 242 { 243 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 244 opcode; 245 } 246 247 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 248 { 249 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 250 251 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 252 (imm4_0 << 7) | opcode; 253 } 254 255 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 256 { 257 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 258 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 259 260 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 261 (imm4_1 << 7) | opcode; 262 } 263 264 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 265 { 266 return (imm31_12 << 12) | (rd << 7) | opcode; 267 } 268 269 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 270 { 271 u32 imm; 272 273 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 274 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 275 276 return (imm << 12) | (rd << 7) | opcode; 277 } 278 279 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 280 u8 funct3, u8 rd, u8 opcode) 281 { 282 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 283 284 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 285 } 286 287 /* RISC-V compressed instruction formats. */ 288 289 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 290 { 291 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 292 } 293 294 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 295 { 296 u32 imm; 297 298 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 299 return (funct3 << 13) | (rd << 7) | op | imm; 300 } 301 302 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 303 { 304 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 305 } 306 307 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 308 { 309 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 310 } 311 312 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 313 u8 op) 314 { 315 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 316 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 317 } 318 319 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 320 u8 op) 321 { 322 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 323 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 324 } 325 326 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 327 { 328 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 329 ((rs2 & 0x7) << 2) | op; 330 } 331 332 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 333 { 334 u32 imm; 335 336 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 337 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 338 } 339 340 /* Instructions shared by both RV32 and RV64. */ 341 342 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 343 { 344 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 345 } 346 347 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 348 { 349 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 350 } 351 352 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 353 { 354 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 355 } 356 357 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 358 { 359 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 360 } 361 362 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 363 { 364 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 365 } 366 367 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 368 { 369 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 370 } 371 372 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 373 { 374 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 375 } 376 377 static inline u32 rv_lui(u8 rd, u32 imm31_12) 378 { 379 return rv_u_insn(imm31_12, rd, 0x37); 380 } 381 382 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 383 { 384 return rv_u_insn(imm31_12, rd, 0x17); 385 } 386 387 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 388 { 389 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 390 } 391 392 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 393 { 394 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 395 } 396 397 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 398 { 399 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 400 } 401 402 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 403 { 404 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 405 } 406 407 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 408 { 409 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 410 } 411 412 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 413 { 414 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 415 } 416 417 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 418 { 419 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 420 } 421 422 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 423 { 424 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 425 } 426 427 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 428 { 429 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 430 } 431 432 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 433 { 434 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 435 } 436 437 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 438 { 439 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 440 } 441 442 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) 443 { 444 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33); 445 } 446 447 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 448 { 449 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 450 } 451 452 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) 453 { 454 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33); 455 } 456 457 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 458 { 459 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 460 } 461 462 static inline u32 rv_jal(u8 rd, u32 imm20_1) 463 { 464 return rv_j_insn(imm20_1, rd, 0x6f); 465 } 466 467 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 468 { 469 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 470 } 471 472 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 473 { 474 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 475 } 476 477 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 478 { 479 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 480 } 481 482 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 483 { 484 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 485 } 486 487 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 488 { 489 return rv_bltu(rs2, rs1, imm12_1); 490 } 491 492 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 493 { 494 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 495 } 496 497 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 498 { 499 return rv_bgeu(rs2, rs1, imm12_1); 500 } 501 502 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 503 { 504 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 505 } 506 507 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 508 { 509 return rv_blt(rs2, rs1, imm12_1); 510 } 511 512 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 513 { 514 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 515 } 516 517 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 518 { 519 return rv_bge(rs2, rs1, imm12_1); 520 } 521 522 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) 523 { 524 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03); 525 } 526 527 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) 528 { 529 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03); 530 } 531 532 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 533 { 534 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 535 } 536 537 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 538 { 539 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 540 } 541 542 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 543 { 544 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 545 } 546 547 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 548 { 549 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 550 } 551 552 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 553 { 554 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 555 } 556 557 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 558 { 559 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 560 } 561 562 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 563 { 564 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 565 } 566 567 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 568 { 569 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 570 } 571 572 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 573 { 574 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 575 } 576 577 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 578 { 579 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 580 } 581 582 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 583 { 584 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 585 } 586 587 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 588 { 589 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 590 } 591 592 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 593 { 594 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 595 } 596 597 static inline u32 rv_fence(u8 pred, u8 succ) 598 { 599 u16 imm11_0 = pred << 4 | succ; 600 601 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 602 } 603 604 static inline u32 rv_nop(void) 605 { 606 return rv_i_insn(0, 0, 0, 0, 0x13); 607 } 608 609 /* RVC instrutions. */ 610 611 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 612 { 613 u32 imm; 614 615 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 616 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 617 return rv_ciw_insn(0x0, imm, rd, 0x0); 618 } 619 620 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 621 { 622 u32 imm_hi, imm_lo; 623 624 imm_hi = (imm7 & 0x38) >> 3; 625 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 626 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 627 } 628 629 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 630 { 631 u32 imm_hi, imm_lo; 632 633 imm_hi = (imm7 & 0x38) >> 3; 634 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 635 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 636 } 637 638 static inline u16 rvc_addi(u8 rd, u32 imm6) 639 { 640 return rv_ci_insn(0, imm6, rd, 0x1); 641 } 642 643 static inline u16 rvc_li(u8 rd, u32 imm6) 644 { 645 return rv_ci_insn(0x2, imm6, rd, 0x1); 646 } 647 648 static inline u16 rvc_addi16sp(u32 imm10) 649 { 650 u32 imm; 651 652 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 653 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 654 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 655 } 656 657 static inline u16 rvc_lui(u8 rd, u32 imm6) 658 { 659 return rv_ci_insn(0x3, imm6, rd, 0x1); 660 } 661 662 static inline u16 rvc_srli(u8 rd, u32 imm6) 663 { 664 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 665 } 666 667 static inline u16 rvc_srai(u8 rd, u32 imm6) 668 { 669 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 670 } 671 672 static inline u16 rvc_andi(u8 rd, u32 imm6) 673 { 674 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 675 } 676 677 static inline u16 rvc_sub(u8 rd, u8 rs) 678 { 679 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 680 } 681 682 static inline u16 rvc_xor(u8 rd, u8 rs) 683 { 684 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 685 } 686 687 static inline u16 rvc_or(u8 rd, u8 rs) 688 { 689 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 690 } 691 692 static inline u16 rvc_and(u8 rd, u8 rs) 693 { 694 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 695 } 696 697 static inline u16 rvc_slli(u8 rd, u32 imm6) 698 { 699 return rv_ci_insn(0, imm6, rd, 0x2); 700 } 701 702 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 703 { 704 u32 imm; 705 706 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 707 return rv_ci_insn(0x2, imm, rd, 0x2); 708 } 709 710 static inline u16 rvc_jr(u8 rs1) 711 { 712 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 713 } 714 715 static inline u16 rvc_mv(u8 rd, u8 rs) 716 { 717 return rv_cr_insn(0x8, rd, rs, 0x2); 718 } 719 720 static inline u16 rvc_jalr(u8 rs1) 721 { 722 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 723 } 724 725 static inline u16 rvc_add(u8 rd, u8 rs) 726 { 727 return rv_cr_insn(0x9, rd, rs, 0x2); 728 } 729 730 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 731 { 732 u32 imm; 733 734 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 735 return rv_css_insn(0x6, imm, rs2, 0x2); 736 } 737 738 /* RVZBB instrutions. */ 739 static inline u32 rvzbb_sextb(u8 rd, u8 rs1) 740 { 741 return rv_i_insn(0x604, rs1, 1, rd, 0x13); 742 } 743 744 static inline u32 rvzbb_sexth(u8 rd, u8 rs1) 745 { 746 return rv_i_insn(0x605, rs1, 1, rd, 0x13); 747 } 748 749 static inline u32 rvzbb_zexth(u8 rd, u8 rs) 750 { 751 if (IS_ENABLED(CONFIG_64BIT)) 752 return rv_i_insn(0x80, rs, 4, rd, 0x3b); 753 754 return rv_i_insn(0x80, rs, 4, rd, 0x33); 755 } 756 757 static inline u32 rvzbb_rev8(u8 rd, u8 rs) 758 { 759 if (IS_ENABLED(CONFIG_64BIT)) 760 return rv_i_insn(0x6b8, rs, 5, rd, 0x13); 761 762 return rv_i_insn(0x698, rs, 5, rd, 0x13); 763 } 764 765 /* 766 * RV64-only instructions. 767 * 768 * These instructions are not available on RV32. Wrap them below a #if to 769 * ensure that the RV32 JIT doesn't emit any of these instructions. 770 */ 771 772 #if __riscv_xlen == 64 773 774 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 775 { 776 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 777 } 778 779 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 780 { 781 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 782 } 783 784 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 785 { 786 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 787 } 788 789 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 790 { 791 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 792 } 793 794 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 795 { 796 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 797 } 798 799 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 800 { 801 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 802 } 803 804 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 805 { 806 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 807 } 808 809 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 810 { 811 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 812 } 813 814 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 815 { 816 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 817 } 818 819 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 820 { 821 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 822 } 823 824 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) 825 { 826 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); 827 } 828 829 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 830 { 831 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 832 } 833 834 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) 835 { 836 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); 837 } 838 839 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 840 { 841 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 842 } 843 844 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 845 { 846 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 847 } 848 849 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 850 { 851 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 852 } 853 854 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 855 { 856 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 857 } 858 859 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 860 { 861 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 862 } 863 864 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 865 { 866 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 867 } 868 869 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 870 { 871 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 872 } 873 874 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 875 { 876 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 877 } 878 879 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 880 { 881 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 882 } 883 884 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 885 { 886 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 887 } 888 889 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 890 { 891 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 892 } 893 894 /* RV64-only RVC instructions. */ 895 896 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 897 { 898 u32 imm_hi, imm_lo; 899 900 imm_hi = (imm8 & 0x38) >> 3; 901 imm_lo = (imm8 & 0xc0) >> 6; 902 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 903 } 904 905 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 906 { 907 u32 imm_hi, imm_lo; 908 909 imm_hi = (imm8 & 0x38) >> 3; 910 imm_lo = (imm8 & 0xc0) >> 6; 911 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 912 } 913 914 static inline u16 rvc_subw(u8 rd, u8 rs) 915 { 916 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 917 } 918 919 static inline u16 rvc_addiw(u8 rd, u32 imm6) 920 { 921 return rv_ci_insn(0x1, imm6, rd, 0x1); 922 } 923 924 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 925 { 926 u32 imm; 927 928 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 929 return rv_ci_insn(0x3, imm, rd, 0x2); 930 } 931 932 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 933 { 934 u32 imm; 935 936 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 937 return rv_css_insn(0x7, imm, rs2, 0x2); 938 } 939 940 #endif /* __riscv_xlen == 64 */ 941 942 /* Helper functions that emit RVC instructions when possible. */ 943 944 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 945 { 946 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 947 emitc(rvc_jalr(rs), ctx); 948 else if (rvc_enabled() && !rd && rs && !imm) 949 emitc(rvc_jr(rs), ctx); 950 else 951 emit(rv_jalr(rd, rs, imm), ctx); 952 } 953 954 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 955 { 956 if (rvc_enabled() && rd && rs) 957 emitc(rvc_mv(rd, rs), ctx); 958 else 959 emit(rv_addi(rd, rs, 0), ctx); 960 } 961 962 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 963 { 964 if (rvc_enabled() && rd && rd == rs1 && rs2) 965 emitc(rvc_add(rd, rs2), ctx); 966 else 967 emit(rv_add(rd, rs1, rs2), ctx); 968 } 969 970 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 971 { 972 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 973 emitc(rvc_addi16sp(imm), ctx); 974 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 975 !(imm & 0x3) && imm) 976 emitc(rvc_addi4spn(rd, imm), ctx); 977 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 978 emitc(rvc_addi(rd, imm), ctx); 979 else 980 emit(rv_addi(rd, rs, imm), ctx); 981 } 982 983 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 984 { 985 if (rvc_enabled() && rd && is_6b_int(imm)) 986 emitc(rvc_li(rd, imm), ctx); 987 else 988 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 989 } 990 991 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 992 { 993 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 994 emitc(rvc_lui(rd, imm), ctx); 995 else 996 emit(rv_lui(rd, imm), ctx); 997 } 998 999 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1000 { 1001 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 1002 emitc(rvc_slli(rd, imm), ctx); 1003 else 1004 emit(rv_slli(rd, rs, imm), ctx); 1005 } 1006 1007 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1008 { 1009 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 1010 emitc(rvc_andi(rd, imm), ctx); 1011 else 1012 emit(rv_andi(rd, rs, imm), ctx); 1013 } 1014 1015 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1016 { 1017 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1018 emitc(rvc_srli(rd, imm), ctx); 1019 else 1020 emit(rv_srli(rd, rs, imm), ctx); 1021 } 1022 1023 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1024 { 1025 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1026 emitc(rvc_srai(rd, imm), ctx); 1027 else 1028 emit(rv_srai(rd, rs, imm), ctx); 1029 } 1030 1031 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1032 { 1033 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1034 emitc(rvc_sub(rd, rs2), ctx); 1035 else 1036 emit(rv_sub(rd, rs1, rs2), ctx); 1037 } 1038 1039 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1040 { 1041 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1042 emitc(rvc_or(rd, rs2), ctx); 1043 else 1044 emit(rv_or(rd, rs1, rs2), ctx); 1045 } 1046 1047 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1048 { 1049 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1050 emitc(rvc_and(rd, rs2), ctx); 1051 else 1052 emit(rv_and(rd, rs1, rs2), ctx); 1053 } 1054 1055 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1056 { 1057 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1058 emitc(rvc_xor(rd, rs2), ctx); 1059 else 1060 emit(rv_xor(rd, rs1, rs2), ctx); 1061 } 1062 1063 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1064 { 1065 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 1066 emitc(rvc_lwsp(rd, off), ctx); 1067 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 1068 emitc(rvc_lw(rd, off, rs1), ctx); 1069 else 1070 emit(rv_lw(rd, off, rs1), ctx); 1071 } 1072 1073 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1074 { 1075 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1076 emitc(rvc_swsp(off, rs2), ctx); 1077 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1078 emitc(rvc_sw(rs1, off, rs2), ctx); 1079 else 1080 emit(rv_sw(rs1, off, rs2), ctx); 1081 } 1082 1083 /* RV64-only helper functions. */ 1084 #if __riscv_xlen == 64 1085 1086 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1087 { 1088 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1089 emitc(rvc_addiw(rd, imm), ctx); 1090 else 1091 emit(rv_addiw(rd, rs, imm), ctx); 1092 } 1093 1094 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1095 { 1096 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1097 emitc(rvc_ldsp(rd, off), ctx); 1098 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1099 emitc(rvc_ld(rd, off, rs1), ctx); 1100 else 1101 emit(rv_ld(rd, off, rs1), ctx); 1102 } 1103 1104 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1105 { 1106 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1107 emitc(rvc_sdsp(off, rs2), ctx); 1108 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1109 emitc(rvc_sd(rs1, off, rs2), ctx); 1110 else 1111 emit(rv_sd(rs1, off, rs2), ctx); 1112 } 1113 1114 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1115 { 1116 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1117 emitc(rvc_subw(rd, rs2), ctx); 1118 else 1119 emit(rv_subw(rd, rs1, rs2), ctx); 1120 } 1121 1122 static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) 1123 { 1124 if (rvzbb_enabled()) { 1125 emit(rvzbb_sextb(rd, rs), ctx); 1126 return; 1127 } 1128 1129 emit_slli(rd, rs, 56, ctx); 1130 emit_srai(rd, rd, 56, ctx); 1131 } 1132 1133 static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1134 { 1135 if (rvzbb_enabled()) { 1136 emit(rvzbb_sexth(rd, rs), ctx); 1137 return; 1138 } 1139 1140 emit_slli(rd, rs, 48, ctx); 1141 emit_srai(rd, rd, 48, ctx); 1142 } 1143 1144 static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1145 { 1146 emit_addiw(rd, rs, 0, ctx); 1147 } 1148 1149 static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1150 { 1151 if (rvzbb_enabled()) { 1152 emit(rvzbb_zexth(rd, rs), ctx); 1153 return; 1154 } 1155 1156 emit_slli(rd, rs, 48, ctx); 1157 emit_srli(rd, rd, 48, ctx); 1158 } 1159 1160 static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1161 { 1162 emit_slli(rd, rs, 32, ctx); 1163 emit_srli(rd, rd, 32, ctx); 1164 } 1165 1166 static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) 1167 { 1168 if (rvzbb_enabled()) { 1169 int bits = 64 - imm; 1170 1171 emit(rvzbb_rev8(rd, rd), ctx); 1172 if (bits) 1173 emit_srli(rd, rd, bits, ctx); 1174 return; 1175 } 1176 1177 emit_li(RV_REG_T2, 0, ctx); 1178 1179 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1180 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1181 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1182 emit_srli(rd, rd, 8, ctx); 1183 if (imm == 16) 1184 goto out_be; 1185 1186 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1187 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1188 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1189 emit_srli(rd, rd, 8, ctx); 1190 1191 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1192 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1193 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1194 emit_srli(rd, rd, 8, ctx); 1195 if (imm == 32) 1196 goto out_be; 1197 1198 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1199 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1200 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1201 emit_srli(rd, rd, 8, ctx); 1202 1203 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1204 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1205 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1206 emit_srli(rd, rd, 8, ctx); 1207 1208 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1209 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1210 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1211 emit_srli(rd, rd, 8, ctx); 1212 1213 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1214 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1215 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1216 emit_srli(rd, rd, 8, ctx); 1217 out_be: 1218 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1219 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1220 1221 emit_mv(rd, RV_REG_T2, ctx); 1222 } 1223 1224 #endif /* __riscv_xlen == 64 */ 1225 1226 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); 1227 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1228 1229 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1230 bool extra_pass); 1231 1232 #endif /* _BPF_JIT_H */ 1233