1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 #if defined(CONFIG_RETPOLINE) && defined(CONFIG_X86) 25 26 static struct static_key_false nf_tables_skip_direct_calls; 27 28 static bool nf_skip_indirect_calls(void) 29 { 30 return static_branch_likely(&nf_tables_skip_direct_calls); 31 } 32 33 static void __init nf_skip_indirect_calls_enable(void) 34 { 35 if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE)) 36 static_branch_enable(&nf_tables_skip_direct_calls); 37 } 38 #else 39 static inline bool nf_skip_indirect_calls(void) { return false; } 40 41 static inline void nf_skip_indirect_calls_enable(void) { } 42 #endif 43 44 static noinline void __nft_trace_packet(struct nft_traceinfo *info, 45 const struct nft_chain *chain, 46 enum nft_trace_types type) 47 { 48 if (!info->trace || !info->nf_trace) 49 return; 50 51 info->chain = chain; 52 info->type = type; 53 54 nft_trace_notify(info); 55 } 56 57 static inline void nft_trace_packet(const struct nft_pktinfo *pkt, 58 struct nft_traceinfo *info, 59 const struct nft_chain *chain, 60 const struct nft_rule_dp *rule, 61 enum nft_trace_types type) 62 { 63 if (static_branch_unlikely(&nft_trace_enabled)) { 64 info->nf_trace = pkt->skb->nf_trace; 65 info->rule = rule; 66 __nft_trace_packet(info, chain, type); 67 } 68 } 69 70 static inline void nft_trace_copy_nftrace(const struct nft_pktinfo *pkt, 71 struct nft_traceinfo *info) 72 { 73 if (static_branch_unlikely(&nft_trace_enabled)) { 74 if (info->trace) 75 info->nf_trace = pkt->skb->nf_trace; 76 } 77 } 78 79 static void nft_bitwise_fast_eval(const struct nft_expr *expr, 80 struct nft_regs *regs) 81 { 82 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); 83 u32 *src = ®s->data[priv->sreg]; 84 u32 *dst = ®s->data[priv->dreg]; 85 86 *dst = (*src & priv->mask) ^ priv->xor; 87 } 88 89 static void nft_cmp_fast_eval(const struct nft_expr *expr, 90 struct nft_regs *regs) 91 { 92 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 93 94 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) 95 return; 96 regs->verdict.code = NFT_BREAK; 97 } 98 99 static void nft_cmp16_fast_eval(const struct nft_expr *expr, 100 struct nft_regs *regs) 101 { 102 const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr); 103 const u64 *reg_data = (const u64 *)®s->data[priv->sreg]; 104 const u64 *mask = (const u64 *)&priv->mask; 105 const u64 *data = (const u64 *)&priv->data; 106 107 if (((reg_data[0] & mask[0]) == data[0] && 108 ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv) 109 return; 110 regs->verdict.code = NFT_BREAK; 111 } 112 113 static noinline void __nft_trace_verdict(struct nft_traceinfo *info, 114 const struct nft_chain *chain, 115 const struct nft_regs *regs) 116 { 117 enum nft_trace_types type; 118 119 switch (regs->verdict.code) { 120 case NFT_CONTINUE: 121 case NFT_RETURN: 122 type = NFT_TRACETYPE_RETURN; 123 break; 124 case NF_STOLEN: 125 type = NFT_TRACETYPE_RULE; 126 /* can't access skb->nf_trace; use copy */ 127 break; 128 default: 129 type = NFT_TRACETYPE_RULE; 130 131 if (info->trace) 132 info->nf_trace = info->pkt->skb->nf_trace; 133 break; 134 } 135 136 __nft_trace_packet(info, chain, type); 137 } 138 139 static inline void nft_trace_verdict(struct nft_traceinfo *info, 140 const struct nft_chain *chain, 141 const struct nft_rule_dp *rule, 142 const struct nft_regs *regs) 143 { 144 if (static_branch_unlikely(&nft_trace_enabled)) { 145 info->rule = rule; 146 __nft_trace_verdict(info, chain, regs); 147 } 148 } 149 150 static bool nft_payload_fast_eval(const struct nft_expr *expr, 151 struct nft_regs *regs, 152 const struct nft_pktinfo *pkt) 153 { 154 const struct nft_payload *priv = nft_expr_priv(expr); 155 const struct sk_buff *skb = pkt->skb; 156 u32 *dest = ®s->data[priv->dreg]; 157 unsigned char *ptr; 158 159 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 160 ptr = skb_network_header(skb); 161 else { 162 if (!(pkt->flags & NFT_PKTINFO_L4PROTO)) 163 return false; 164 ptr = skb_network_header(skb) + nft_thoff(pkt); 165 } 166 167 ptr += priv->offset; 168 169 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 170 return false; 171 172 *dest = 0; 173 if (priv->len == 2) 174 *(u16 *)dest = *(u16 *)ptr; 175 else if (priv->len == 4) 176 *(u32 *)dest = *(u32 *)ptr; 177 else 178 *(u8 *)dest = *(u8 *)ptr; 179 return true; 180 } 181 182 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 183 184 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 185 const struct nft_pktinfo *pkt) 186 { 187 struct nft_base_chain *base_chain; 188 struct nft_stats __percpu *pstats; 189 struct nft_stats *stats; 190 191 base_chain = nft_base_chain(chain); 192 193 pstats = READ_ONCE(base_chain->stats); 194 if (pstats) { 195 local_bh_disable(); 196 stats = this_cpu_ptr(pstats); 197 u64_stats_update_begin(&stats->syncp); 198 stats->pkts++; 199 stats->bytes += pkt->skb->len; 200 u64_stats_update_end(&stats->syncp); 201 local_bh_enable(); 202 } 203 } 204 205 struct nft_jumpstack { 206 const struct nft_chain *chain; 207 const struct nft_rule_dp *rule; 208 const struct nft_rule_dp *last_rule; 209 }; 210 211 static void expr_call_ops_eval(const struct nft_expr *expr, 212 struct nft_regs *regs, 213 struct nft_pktinfo *pkt) 214 { 215 #ifdef CONFIG_RETPOLINE 216 unsigned long e; 217 218 if (nf_skip_indirect_calls()) 219 goto indirect_call; 220 221 e = (unsigned long)expr->ops->eval; 222 #define X(e, fun) \ 223 do { if ((e) == (unsigned long)(fun)) \ 224 return fun(expr, regs, pkt); } while (0) 225 226 X(e, nft_payload_eval); 227 X(e, nft_cmp_eval); 228 X(e, nft_counter_eval); 229 X(e, nft_meta_get_eval); 230 X(e, nft_lookup_eval); 231 #if IS_ENABLED(CONFIG_NFT_CT) 232 X(e, nft_ct_get_fast_eval); 233 #endif 234 X(e, nft_range_eval); 235 X(e, nft_immediate_eval); 236 X(e, nft_byteorder_eval); 237 X(e, nft_dynset_eval); 238 X(e, nft_rt_get_eval); 239 X(e, nft_bitwise_eval); 240 X(e, nft_objref_eval); 241 X(e, nft_objref_map_eval); 242 #undef X 243 indirect_call: 244 #endif /* CONFIG_RETPOLINE */ 245 expr->ops->eval(expr, regs, pkt); 246 } 247 248 #define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0] 249 #define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size 250 #define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen] 251 #define nft_rule_next(rule) (void *)rule + sizeof(*rule) + rule->dlen 252 253 #define nft_rule_dp_for_each_expr(expr, last, rule) \ 254 for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \ 255 (expr) != (last); \ 256 (expr) = nft_rule_expr_next(expr)) 257 258 unsigned int 259 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 260 { 261 const struct nft_chain *chain = priv, *basechain = chain; 262 const struct nft_rule_dp *rule, *last_rule; 263 const struct net *net = nft_net(pkt); 264 const struct nft_expr *expr, *last; 265 struct nft_regs regs = {}; 266 unsigned int stackptr = 0; 267 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 268 bool genbit = READ_ONCE(net->nft.gencursor); 269 struct nft_rule_blob *blob; 270 struct nft_traceinfo info; 271 272 info.trace = false; 273 if (static_branch_unlikely(&nft_trace_enabled)) 274 nft_trace_init(&info, pkt, ®s.verdict, basechain); 275 do_chain: 276 if (genbit) 277 blob = rcu_dereference(chain->blob_gen_1); 278 else 279 blob = rcu_dereference(chain->blob_gen_0); 280 281 rule = (struct nft_rule_dp *)blob->data; 282 last_rule = (void *)blob->data + blob->size; 283 next_rule: 284 regs.verdict.code = NFT_CONTINUE; 285 for (; rule < last_rule; rule = nft_rule_next(rule)) { 286 nft_rule_dp_for_each_expr(expr, last, rule) { 287 if (expr->ops == &nft_cmp_fast_ops) 288 nft_cmp_fast_eval(expr, ®s); 289 else if (expr->ops == &nft_cmp16_fast_ops) 290 nft_cmp16_fast_eval(expr, ®s); 291 else if (expr->ops == &nft_bitwise_fast_ops) 292 nft_bitwise_fast_eval(expr, ®s); 293 else if (expr->ops != &nft_payload_fast_ops || 294 !nft_payload_fast_eval(expr, ®s, pkt)) 295 expr_call_ops_eval(expr, ®s, pkt); 296 297 if (regs.verdict.code != NFT_CONTINUE) 298 break; 299 } 300 301 switch (regs.verdict.code) { 302 case NFT_BREAK: 303 regs.verdict.code = NFT_CONTINUE; 304 nft_trace_copy_nftrace(pkt, &info); 305 continue; 306 case NFT_CONTINUE: 307 nft_trace_packet(pkt, &info, chain, rule, 308 NFT_TRACETYPE_RULE); 309 continue; 310 } 311 break; 312 } 313 314 nft_trace_verdict(&info, chain, rule, ®s); 315 316 switch (regs.verdict.code & NF_VERDICT_MASK) { 317 case NF_ACCEPT: 318 case NF_DROP: 319 case NF_QUEUE: 320 case NF_STOLEN: 321 return regs.verdict.code; 322 } 323 324 switch (regs.verdict.code) { 325 case NFT_JUMP: 326 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 327 return NF_DROP; 328 jumpstack[stackptr].chain = chain; 329 jumpstack[stackptr].rule = nft_rule_next(rule); 330 jumpstack[stackptr].last_rule = last_rule; 331 stackptr++; 332 fallthrough; 333 case NFT_GOTO: 334 chain = regs.verdict.chain; 335 goto do_chain; 336 case NFT_CONTINUE: 337 case NFT_RETURN: 338 break; 339 default: 340 WARN_ON_ONCE(1); 341 } 342 343 if (stackptr > 0) { 344 stackptr--; 345 chain = jumpstack[stackptr].chain; 346 rule = jumpstack[stackptr].rule; 347 last_rule = jumpstack[stackptr].last_rule; 348 goto next_rule; 349 } 350 351 nft_trace_packet(pkt, &info, basechain, NULL, NFT_TRACETYPE_POLICY); 352 353 if (static_branch_unlikely(&nft_counters_enabled)) 354 nft_update_chain_stats(basechain, pkt); 355 356 return nft_base_chain(basechain)->policy; 357 } 358 EXPORT_SYMBOL_GPL(nft_do_chain); 359 360 static struct nft_expr_type *nft_basic_types[] = { 361 &nft_imm_type, 362 &nft_cmp_type, 363 &nft_lookup_type, 364 &nft_bitwise_type, 365 &nft_byteorder_type, 366 &nft_payload_type, 367 &nft_dynset_type, 368 &nft_range_type, 369 &nft_meta_type, 370 &nft_rt_type, 371 &nft_exthdr_type, 372 &nft_last_type, 373 &nft_counter_type, 374 &nft_objref_type, 375 &nft_inner_type, 376 }; 377 378 static struct nft_object_type *nft_basic_objects[] = { 379 #ifdef CONFIG_NETWORK_SECMARK 380 &nft_secmark_obj_type, 381 #endif 382 &nft_counter_obj_type, 383 }; 384 385 int __init nf_tables_core_module_init(void) 386 { 387 int err, i, j = 0; 388 389 nft_counter_init_seqcount(); 390 391 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 392 err = nft_register_obj(nft_basic_objects[i]); 393 if (err) 394 goto err; 395 } 396 397 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 398 err = nft_register_expr(nft_basic_types[j]); 399 if (err) 400 goto err; 401 } 402 403 nf_skip_indirect_calls_enable(); 404 405 return 0; 406 407 err: 408 while (j-- > 0) 409 nft_unregister_expr(nft_basic_types[j]); 410 411 while (i-- > 0) 412 nft_unregister_obj(nft_basic_objects[i]); 413 414 return err; 415 } 416 417 void nf_tables_core_module_exit(void) 418 { 419 int i; 420 421 i = ARRAY_SIZE(nft_basic_types); 422 while (i-- > 0) 423 nft_unregister_expr(nft_basic_types[i]); 424 425 i = ARRAY_SIZE(nft_basic_objects); 426 while (i-- > 0) 427 nft_unregister_obj(nft_basic_objects[i]); 428 } 429