1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/skbuff.h> 14 #include <linux/netlink.h> 15 #include <linux/netfilter.h> 16 #include <linux/static_key.h> 17 #include <linux/netfilter/nfnetlink.h> 18 #include <linux/netfilter/nf_tables.h> 19 #include <net/netfilter/nf_tables_core.h> 20 #include <net/netfilter/nf_tables.h> 21 #include <net/netfilter/nf_log.h> 22 #include <net/netfilter/nft_meta.h> 23 24 static noinline void __nft_trace_packet(struct nft_traceinfo *info, 25 const struct nft_chain *chain, 26 enum nft_trace_types type) 27 { 28 const struct nft_pktinfo *pkt = info->pkt; 29 30 if (!info->trace || !pkt->skb->nf_trace) 31 return; 32 33 info->chain = chain; 34 info->type = type; 35 36 nft_trace_notify(info); 37 } 38 39 static inline void nft_trace_packet(struct nft_traceinfo *info, 40 const struct nft_chain *chain, 41 const struct nft_rule_dp *rule, 42 enum nft_trace_types type) 43 { 44 if (static_branch_unlikely(&nft_trace_enabled)) { 45 info->rule = rule; 46 __nft_trace_packet(info, chain, type); 47 } 48 } 49 50 static void nft_bitwise_fast_eval(const struct nft_expr *expr, 51 struct nft_regs *regs) 52 { 53 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); 54 u32 *src = ®s->data[priv->sreg]; 55 u32 *dst = ®s->data[priv->dreg]; 56 57 *dst = (*src & priv->mask) ^ priv->xor; 58 } 59 60 static void nft_cmp_fast_eval(const struct nft_expr *expr, 61 struct nft_regs *regs) 62 { 63 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); 64 65 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) 66 return; 67 regs->verdict.code = NFT_BREAK; 68 } 69 70 static void nft_cmp16_fast_eval(const struct nft_expr *expr, 71 struct nft_regs *regs) 72 { 73 const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr); 74 const u64 *reg_data = (const u64 *)®s->data[priv->sreg]; 75 const u64 *mask = (const u64 *)&priv->mask; 76 const u64 *data = (const u64 *)&priv->data; 77 78 if (((reg_data[0] & mask[0]) == data[0] && 79 ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv) 80 return; 81 regs->verdict.code = NFT_BREAK; 82 } 83 84 static noinline void __nft_trace_verdict(struct nft_traceinfo *info, 85 const struct nft_chain *chain, 86 const struct nft_regs *regs) 87 { 88 enum nft_trace_types type; 89 90 switch (regs->verdict.code) { 91 case NFT_CONTINUE: 92 case NFT_RETURN: 93 type = NFT_TRACETYPE_RETURN; 94 break; 95 default: 96 type = NFT_TRACETYPE_RULE; 97 break; 98 } 99 100 __nft_trace_packet(info, chain, type); 101 } 102 103 static inline void nft_trace_verdict(struct nft_traceinfo *info, 104 const struct nft_chain *chain, 105 const struct nft_rule_dp *rule, 106 const struct nft_regs *regs) 107 { 108 if (static_branch_unlikely(&nft_trace_enabled)) { 109 info->rule = rule; 110 __nft_trace_verdict(info, chain, regs); 111 } 112 } 113 114 static bool nft_payload_fast_eval(const struct nft_expr *expr, 115 struct nft_regs *regs, 116 const struct nft_pktinfo *pkt) 117 { 118 const struct nft_payload *priv = nft_expr_priv(expr); 119 const struct sk_buff *skb = pkt->skb; 120 u32 *dest = ®s->data[priv->dreg]; 121 unsigned char *ptr; 122 123 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) 124 ptr = skb_network_header(skb); 125 else { 126 if (!(pkt->flags & NFT_PKTINFO_L4PROTO)) 127 return false; 128 ptr = skb_network_header(skb) + nft_thoff(pkt); 129 } 130 131 ptr += priv->offset; 132 133 if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) 134 return false; 135 136 *dest = 0; 137 if (priv->len == 2) 138 *(u16 *)dest = *(u16 *)ptr; 139 else if (priv->len == 4) 140 *(u32 *)dest = *(u32 *)ptr; 141 else 142 *(u8 *)dest = *(u8 *)ptr; 143 return true; 144 } 145 146 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); 147 148 static noinline void nft_update_chain_stats(const struct nft_chain *chain, 149 const struct nft_pktinfo *pkt) 150 { 151 struct nft_base_chain *base_chain; 152 struct nft_stats __percpu *pstats; 153 struct nft_stats *stats; 154 155 base_chain = nft_base_chain(chain); 156 157 pstats = READ_ONCE(base_chain->stats); 158 if (pstats) { 159 local_bh_disable(); 160 stats = this_cpu_ptr(pstats); 161 u64_stats_update_begin(&stats->syncp); 162 stats->pkts++; 163 stats->bytes += pkt->skb->len; 164 u64_stats_update_end(&stats->syncp); 165 local_bh_enable(); 166 } 167 } 168 169 struct nft_jumpstack { 170 const struct nft_chain *chain; 171 const struct nft_rule_dp *rule; 172 const struct nft_rule_dp *last_rule; 173 }; 174 175 static void expr_call_ops_eval(const struct nft_expr *expr, 176 struct nft_regs *regs, 177 struct nft_pktinfo *pkt) 178 { 179 #ifdef CONFIG_RETPOLINE 180 unsigned long e = (unsigned long)expr->ops->eval; 181 #define X(e, fun) \ 182 do { if ((e) == (unsigned long)(fun)) \ 183 return fun(expr, regs, pkt); } while (0) 184 185 X(e, nft_payload_eval); 186 X(e, nft_cmp_eval); 187 X(e, nft_counter_eval); 188 X(e, nft_meta_get_eval); 189 X(e, nft_lookup_eval); 190 X(e, nft_range_eval); 191 X(e, nft_immediate_eval); 192 X(e, nft_byteorder_eval); 193 X(e, nft_dynset_eval); 194 X(e, nft_rt_get_eval); 195 X(e, nft_bitwise_eval); 196 #undef X 197 #endif /* CONFIG_RETPOLINE */ 198 expr->ops->eval(expr, regs, pkt); 199 } 200 201 #define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0] 202 #define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size 203 #define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen] 204 #define nft_rule_next(rule) (void *)rule + sizeof(*rule) + rule->dlen 205 206 #define nft_rule_dp_for_each_expr(expr, last, rule) \ 207 for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \ 208 (expr) != (last); \ 209 (expr) = nft_rule_expr_next(expr)) 210 211 unsigned int 212 nft_do_chain(struct nft_pktinfo *pkt, void *priv) 213 { 214 const struct nft_chain *chain = priv, *basechain = chain; 215 const struct nft_rule_dp *rule, *last_rule; 216 const struct net *net = nft_net(pkt); 217 const struct nft_expr *expr, *last; 218 struct nft_regs regs = {}; 219 unsigned int stackptr = 0; 220 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; 221 bool genbit = READ_ONCE(net->nft.gencursor); 222 struct nft_rule_blob *blob; 223 struct nft_traceinfo info; 224 225 info.trace = false; 226 if (static_branch_unlikely(&nft_trace_enabled)) 227 nft_trace_init(&info, pkt, ®s.verdict, basechain); 228 do_chain: 229 if (genbit) 230 blob = rcu_dereference(chain->blob_gen_1); 231 else 232 blob = rcu_dereference(chain->blob_gen_0); 233 234 rule = (struct nft_rule_dp *)blob->data; 235 last_rule = (void *)blob->data + blob->size; 236 next_rule: 237 regs.verdict.code = NFT_CONTINUE; 238 for (; rule < last_rule; rule = nft_rule_next(rule)) { 239 nft_rule_dp_for_each_expr(expr, last, rule) { 240 if (expr->ops == &nft_cmp_fast_ops) 241 nft_cmp_fast_eval(expr, ®s); 242 else if (expr->ops == &nft_cmp16_fast_ops) 243 nft_cmp16_fast_eval(expr, ®s); 244 else if (expr->ops == &nft_bitwise_fast_ops) 245 nft_bitwise_fast_eval(expr, ®s); 246 else if (expr->ops != &nft_payload_fast_ops || 247 !nft_payload_fast_eval(expr, ®s, pkt)) 248 expr_call_ops_eval(expr, ®s, pkt); 249 250 if (regs.verdict.code != NFT_CONTINUE) 251 break; 252 } 253 254 switch (regs.verdict.code) { 255 case NFT_BREAK: 256 regs.verdict.code = NFT_CONTINUE; 257 continue; 258 case NFT_CONTINUE: 259 nft_trace_packet(&info, chain, rule, 260 NFT_TRACETYPE_RULE); 261 continue; 262 } 263 break; 264 } 265 266 nft_trace_verdict(&info, chain, rule, ®s); 267 268 switch (regs.verdict.code & NF_VERDICT_MASK) { 269 case NF_ACCEPT: 270 case NF_DROP: 271 case NF_QUEUE: 272 case NF_STOLEN: 273 return regs.verdict.code; 274 } 275 276 switch (regs.verdict.code) { 277 case NFT_JUMP: 278 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) 279 return NF_DROP; 280 jumpstack[stackptr].chain = chain; 281 jumpstack[stackptr].rule = nft_rule_next(rule); 282 jumpstack[stackptr].last_rule = last_rule; 283 stackptr++; 284 fallthrough; 285 case NFT_GOTO: 286 chain = regs.verdict.chain; 287 goto do_chain; 288 case NFT_CONTINUE: 289 case NFT_RETURN: 290 break; 291 default: 292 WARN_ON_ONCE(1); 293 } 294 295 if (stackptr > 0) { 296 stackptr--; 297 chain = jumpstack[stackptr].chain; 298 rule = jumpstack[stackptr].rule; 299 last_rule = jumpstack[stackptr].last_rule; 300 goto next_rule; 301 } 302 303 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY); 304 305 if (static_branch_unlikely(&nft_counters_enabled)) 306 nft_update_chain_stats(basechain, pkt); 307 308 return nft_base_chain(basechain)->policy; 309 } 310 EXPORT_SYMBOL_GPL(nft_do_chain); 311 312 static struct nft_expr_type *nft_basic_types[] = { 313 &nft_imm_type, 314 &nft_cmp_type, 315 &nft_lookup_type, 316 &nft_bitwise_type, 317 &nft_byteorder_type, 318 &nft_payload_type, 319 &nft_dynset_type, 320 &nft_range_type, 321 &nft_meta_type, 322 &nft_rt_type, 323 &nft_exthdr_type, 324 &nft_last_type, 325 &nft_counter_type, 326 }; 327 328 static struct nft_object_type *nft_basic_objects[] = { 329 #ifdef CONFIG_NETWORK_SECMARK 330 &nft_secmark_obj_type, 331 #endif 332 &nft_counter_obj_type, 333 }; 334 335 int __init nf_tables_core_module_init(void) 336 { 337 int err, i, j = 0; 338 339 nft_counter_init_seqcount(); 340 341 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { 342 err = nft_register_obj(nft_basic_objects[i]); 343 if (err) 344 goto err; 345 } 346 347 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { 348 err = nft_register_expr(nft_basic_types[j]); 349 if (err) 350 goto err; 351 } 352 353 return 0; 354 355 err: 356 while (j-- > 0) 357 nft_unregister_expr(nft_basic_types[j]); 358 359 while (i-- > 0) 360 nft_unregister_obj(nft_basic_objects[i]); 361 362 return err; 363 } 364 365 void nf_tables_core_module_exit(void) 366 { 367 int i; 368 369 i = ARRAY_SIZE(nft_basic_types); 370 while (i-- > 0) 371 nft_unregister_expr(nft_basic_types[i]); 372 373 i = ARRAY_SIZE(nft_basic_objects); 374 while (i-- > 0) 375 nft_unregister_obj(nft_basic_objects[i]); 376 } 377