1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 #if defined(CONFIG_RETPOLINE) && defined(CONFIG_X86) 25 26 static struct static_key_false nf_tables_skip_direct_calls; 27 28 static bool nf_skip_indirect_calls(void) 29 { 30 return static_branch_likely(&nf_tables_skip_direct_calls); 31 } 32 33 static void __init nf_skip_indirect_calls_enable(void) 34 { 35 if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE)) 36 static_branch_enable(&nf_tables_skip_direct_calls); 37 } 38 #else 39 static inline bool nf_skip_indirect_calls(void) { return false; } 40 41 static inline void nf_skip_indirect_calls_enable(void) { } 42 #endif 43 44 static noinline void __nft_trace_packet(const struct nft_pktinfo *pkt, 45 const struct nft_verdict *verdict, 46 const struct nft_rule_dp *rule, 47 struct nft_traceinfo *info, 48 enum nft_trace_types type) 49 { 50 if (!info->trace || !info->nf_trace) 51 return; 52 53 info->type = type; 54 55 nft_trace_notify(pkt, verdict, rule, info); 56 } 57 58 static inline void nft_trace_packet(const struct nft_pktinfo *pkt, 59 struct nft_verdict *verdict, 60 struct nft_traceinfo *info, 61 const struct nft_rule_dp *rule, 62 enum nft_trace_types type) 63 { 64 if (static_branch_unlikely(&nft_trace_enabled)) { 65 info->nf_trace = pkt->skb->nf_trace; 66 __nft_trace_packet(pkt, verdict, rule, info, type); 67 } 68 } 69 70 static inline void nft_trace_copy_nftrace(const struct nft_pktinfo *pkt, 71 struct nft_traceinfo *info) 72 { 73 if (static_branch_unlikely(&nft_trace_enabled)) 74 info->nf_trace = pkt->skb->nf_trace; 75 } 76 77 static void nft_bitwise_fast_eval(const struct nft_expr *expr, 78 struct nft_regs *regs) 79 { 80 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); 81 u32 *src = ®s->data[priv->sreg]; 82 u32 *dst = ®s->data[priv->dreg]; 83 84 *dst = (*src & priv->mask) ^ priv->xor; 85 } 86 87 static void nft_cmp_fast_eval(const struct nft_expr *expr, 88 struct nft_regs *regs) 89 { 90 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 91 92 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) 93 return; 94 regs->verdict.code = NFT_BREAK; 95 } 96 97 static void nft_cmp16_fast_eval(const struct nft_expr *expr, 98 struct nft_regs *regs) 99 { 100 const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr); 101 const u64 *reg_data = (const u64 *)®s->data[priv->sreg]; 102 const u64 *mask = (const u64 *)&priv->mask; 103 const u64 *data = (const u64 *)&priv->data; 104 105 if (((reg_data[0] & mask[0]) == data[0] && 106 ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv) 107 return; 108 regs->verdict.code = NFT_BREAK; 109 } 110 111 static noinline void __nft_trace_verdict(const struct nft_pktinfo *pkt, 112 struct nft_traceinfo *info, 113 const struct nft_rule_dp *rule, 114 const struct nft_regs *regs) 115 { 116 enum nft_trace_types type; 117 118 switch (regs->verdict.code & NF_VERDICT_MASK) { 119 case NFT_CONTINUE: 120 case NFT_RETURN: 121 type = NFT_TRACETYPE_RETURN; 122 break; 123 case NF_STOLEN: 124 type = NFT_TRACETYPE_RULE; 125 /* can't access skb->nf_trace; use copy */ 126 break; 127 default: 128 type = NFT_TRACETYPE_RULE; 129 130 if (info->trace) 131 info->nf_trace = pkt->skb->nf_trace; 132 break; 133 } 134 135 __nft_trace_packet(pkt, ®s->verdict, rule, info, type); 136 } 137 138 static inline void nft_trace_verdict(const struct nft_pktinfo *pkt, 139 struct nft_traceinfo *info, 140 const struct nft_rule_dp *rule, 141 const struct nft_regs *regs) 142 { 143 if (static_branch_unlikely(&nft_trace_enabled)) 144 __nft_trace_verdict(pkt, info, rule, regs); 145 } 146 147 static bool nft_payload_fast_eval(const struct nft_expr *expr, 148 struct nft_regs *regs, 149 const struct nft_pktinfo *pkt) 150 { 151 const struct nft_payload *priv = nft_expr_priv(expr); 152 const struct sk_buff *skb = pkt->skb; 153 u32 *dest = ®s->data[priv->dreg]; 154 unsigned char *ptr; 155 156 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 157 ptr = skb_network_header(skb); 158 else { 159 if (!(pkt->flags & NFT_PKTINFO_L4PROTO)) 160 return false; 161 ptr = skb_network_header(skb) + nft_thoff(pkt); 162 } 163 164 ptr += priv->offset; 165 166 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 167 return false; 168 169 *dest = 0; 170 if (priv->len == 2) 171 *(u16 *)dest = *(u16 *)ptr; 172 else if (priv->len == 4) 173 *(u32 *)dest = *(u32 *)ptr; 174 else 175 *(u8 *)dest = *(u8 *)ptr; 176 return true; 177 } 178 179 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 180 181 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 182 const struct nft_pktinfo *pkt) 183 { 184 struct nft_base_chain *base_chain; 185 struct nft_stats __percpu *pstats; 186 struct nft_stats *stats; 187 188 base_chain = nft_base_chain(chain); 189 190 pstats = READ_ONCE(base_chain->stats); 191 if (pstats) { 192 local_bh_disable(); 193 stats = this_cpu_ptr(pstats); 194 u64_stats_update_begin(&stats->syncp); 195 stats->pkts++; 196 stats->bytes += pkt->skb->len; 197 u64_stats_update_end(&stats->syncp); 198 local_bh_enable(); 199 } 200 } 201 202 struct nft_jumpstack { 203 const struct nft_rule_dp *rule; 204 }; 205 206 static void expr_call_ops_eval(const struct nft_expr *expr, 207 struct nft_regs *regs, 208 struct nft_pktinfo *pkt) 209 { 210 #ifdef CONFIG_RETPOLINE 211 unsigned long e; 212 213 if (nf_skip_indirect_calls()) 214 goto indirect_call; 215 216 e = (unsigned long)expr->ops->eval; 217 #define X(e, fun) \ 218 do { if ((e) == (unsigned long)(fun)) \ 219 return fun(expr, regs, pkt); } while (0) 220 221 X(e, nft_payload_eval); 222 X(e, nft_cmp_eval); 223 X(e, nft_counter_eval); 224 X(e, nft_meta_get_eval); 225 X(e, nft_lookup_eval); 226 #if IS_ENABLED(CONFIG_NFT_CT) 227 X(e, nft_ct_get_fast_eval); 228 #endif 229 X(e, nft_range_eval); 230 X(e, nft_immediate_eval); 231 X(e, nft_byteorder_eval); 232 X(e, nft_dynset_eval); 233 X(e, nft_rt_get_eval); 234 X(e, nft_bitwise_eval); 235 X(e, nft_objref_eval); 236 X(e, nft_objref_map_eval); 237 #undef X 238 indirect_call: 239 #endif /* CONFIG_RETPOLINE */ 240 expr->ops->eval(expr, regs, pkt); 241 } 242 243 #define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0] 244 #define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size 245 #define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen] 246 247 #define nft_rule_dp_for_each_expr(expr, last, rule) \ 248 for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \ 249 (expr) != (last); \ 250 (expr) = nft_rule_expr_next(expr)) 251 252 unsigned int 253 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 254 { 255 const struct nft_chain *chain = priv, *basechain = chain; 256 const struct net *net = nft_net(pkt); 257 const struct nft_expr *expr, *last; 258 const struct nft_rule_dp *rule; 259 struct nft_regs regs = {}; 260 unsigned int stackptr = 0; 261 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 262 bool genbit = READ_ONCE(net->nft.gencursor); 263 struct nft_rule_blob *blob; 264 struct nft_traceinfo info; 265 266 info.trace = false; 267 if (static_branch_unlikely(&nft_trace_enabled)) 268 nft_trace_init(&info, pkt, basechain); 269 do_chain: 270 if (genbit) 271 blob = rcu_dereference(chain->blob_gen_1); 272 else 273 blob = rcu_dereference(chain->blob_gen_0); 274 275 rule = (struct nft_rule_dp *)blob->data; 276 next_rule: 277 regs.verdict.code = NFT_CONTINUE; 278 for (; !rule->is_last ; rule = nft_rule_next(rule)) { 279 nft_rule_dp_for_each_expr(expr, last, rule) { 280 if (expr->ops == &nft_cmp_fast_ops) 281 nft_cmp_fast_eval(expr, ®s); 282 else if (expr->ops == &nft_cmp16_fast_ops) 283 nft_cmp16_fast_eval(expr, ®s); 284 else if (expr->ops == &nft_bitwise_fast_ops) 285 nft_bitwise_fast_eval(expr, ®s); 286 else if (expr->ops != &nft_payload_fast_ops || 287 !nft_payload_fast_eval(expr, ®s, pkt)) 288 expr_call_ops_eval(expr, ®s, pkt); 289 290 if (regs.verdict.code != NFT_CONTINUE) 291 break; 292 } 293 294 switch (regs.verdict.code) { 295 case NFT_BREAK: 296 regs.verdict.code = NFT_CONTINUE; 297 nft_trace_copy_nftrace(pkt, &info); 298 continue; 299 case NFT_CONTINUE: 300 nft_trace_packet(pkt, ®s.verdict, &info, rule, 301 NFT_TRACETYPE_RULE); 302 continue; 303 } 304 break; 305 } 306 307 nft_trace_verdict(pkt, &info, rule, ®s); 308 309 switch (regs.verdict.code & NF_VERDICT_MASK) { 310 case NF_ACCEPT: 311 case NF_QUEUE: 312 case NF_STOLEN: 313 return regs.verdict.code; 314 case NF_DROP: 315 return NF_DROP_REASON(pkt->skb, SKB_DROP_REASON_NETFILTER_DROP, EPERM); 316 } 317 318 switch (regs.verdict.code) { 319 case NFT_JUMP: 320 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 321 return NF_DROP; 322 jumpstack[stackptr].rule = nft_rule_next(rule); 323 stackptr++; 324 fallthrough; 325 case NFT_GOTO: 326 chain = regs.verdict.chain; 327 goto do_chain; 328 case NFT_CONTINUE: 329 case NFT_RETURN: 330 break; 331 default: 332 WARN_ON_ONCE(1); 333 } 334 335 if (stackptr > 0) { 336 stackptr--; 337 rule = jumpstack[stackptr].rule; 338 goto next_rule; 339 } 340 341 nft_trace_packet(pkt, ®s.verdict, &info, NULL, NFT_TRACETYPE_POLICY); 342 343 if (static_branch_unlikely(&nft_counters_enabled)) 344 nft_update_chain_stats(basechain, pkt); 345 346 if (nft_base_chain(basechain)->policy == NF_DROP) 347 return NF_DROP_REASON(pkt->skb, SKB_DROP_REASON_NETFILTER_DROP, EPERM); 348 349 return nft_base_chain(basechain)->policy; 350 } 351 EXPORT_SYMBOL_GPL(nft_do_chain); 352 353 static struct nft_expr_type *nft_basic_types[] = { 354 &nft_imm_type, 355 &nft_cmp_type, 356 &nft_lookup_type, 357 &nft_bitwise_type, 358 &nft_byteorder_type, 359 &nft_payload_type, 360 &nft_dynset_type, 361 &nft_range_type, 362 &nft_meta_type, 363 &nft_rt_type, 364 &nft_exthdr_type, 365 &nft_last_type, 366 &nft_counter_type, 367 &nft_objref_type, 368 &nft_inner_type, 369 }; 370 371 static struct nft_object_type *nft_basic_objects[] = { 372 #ifdef CONFIG_NETWORK_SECMARK 373 &nft_secmark_obj_type, 374 #endif 375 &nft_counter_obj_type, 376 }; 377 378 int __init nf_tables_core_module_init(void) 379 { 380 int err, i, j = 0; 381 382 nft_counter_init_seqcount(); 383 384 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 385 err = nft_register_obj(nft_basic_objects[i]); 386 if (err) 387 goto err; 388 } 389 390 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 391 err = nft_register_expr(nft_basic_types[j]); 392 if (err) 393 goto err; 394 } 395 396 nf_skip_indirect_calls_enable(); 397 398 return 0; 399 400 err: 401 while (j-- > 0) 402 nft_unregister_expr(nft_basic_types[j]); 403 404 while (i-- > 0) 405 nft_unregister_obj(nft_basic_objects[i]); 406 407 return err; 408 } 409 410 void nf_tables_core_module_exit(void) 411 { 412 int i; 413 414 i = ARRAY_SIZE(nft_basic_types); 415 while (i-- > 0) 416 nft_unregister_expr(nft_basic_types[i]); 417 418 i = ARRAY_SIZE(nft_basic_objects); 419 while (i-- > 0) 420 nft_unregister_obj(nft_basic_objects[i]); 421 } 422