1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * xfrm_state.c 4 * 5 * Changes: 6 * Mitsuru KANDA @USAGI 7 * Kazunori MIYAZAWA @USAGI 8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 9 * IPv6 support 10 * YOSHIFUJI Hideaki @USAGI 11 * Split up af-specific functions 12 * Derek Atkins <derek@ihtfp.com> 13 * Add UDP Encapsulation 14 * 15 */ 16 17 #include <linux/workqueue.h> 18 #include <net/xfrm.h> 19 #include <linux/pfkeyv2.h> 20 #include <linux/ipsec.h> 21 #include <linux/module.h> 22 #include <linux/cache.h> 23 #include <linux/audit.h> 24 #include <linux/uaccess.h> 25 #include <linux/ktime.h> 26 #include <linux/slab.h> 27 #include <linux/interrupt.h> 28 #include <linux/kernel.h> 29 30 #include "xfrm_hash.h" 31 32 #define xfrm_state_deref_prot(table, net) \ 33 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock)) 34 35 static void xfrm_state_gc_task(struct work_struct *work); 36 37 /* Each xfrm_state may be linked to two tables: 38 39 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl) 40 2. Hash table by (daddr,family,reqid) to find what SAs exist for given 41 destination/tunnel endpoint. (output) 42 */ 43 44 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024; 45 static __read_mostly seqcount_t xfrm_state_hash_generation = SEQCNT_ZERO(xfrm_state_hash_generation); 46 static struct kmem_cache *xfrm_state_cache __ro_after_init; 47 48 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); 49 static HLIST_HEAD(xfrm_state_gc_list); 50 51 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x) 52 { 53 return refcount_inc_not_zero(&x->refcnt); 54 } 55 56 static inline unsigned int xfrm_dst_hash(struct net *net, 57 const xfrm_address_t *daddr, 58 const xfrm_address_t *saddr, 59 u32 reqid, 60 unsigned short family) 61 { 62 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask); 63 } 64 65 static inline unsigned int xfrm_src_hash(struct net *net, 66 const xfrm_address_t *daddr, 67 const xfrm_address_t *saddr, 68 unsigned short family) 69 { 70 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask); 71 } 72 73 static inline unsigned int 74 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr, 75 __be32 spi, u8 proto, unsigned short family) 76 { 77 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask); 78 } 79 80 static void xfrm_hash_transfer(struct hlist_head *list, 81 struct hlist_head *ndsttable, 82 struct hlist_head *nsrctable, 83 struct hlist_head *nspitable, 84 unsigned int nhashmask) 85 { 86 struct hlist_node *tmp; 87 struct xfrm_state *x; 88 89 hlist_for_each_entry_safe(x, tmp, list, bydst) { 90 unsigned int h; 91 92 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, 93 x->props.reqid, x->props.family, 94 nhashmask); 95 hlist_add_head_rcu(&x->bydst, ndsttable + h); 96 97 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, 98 x->props.family, 99 nhashmask); 100 hlist_add_head_rcu(&x->bysrc, nsrctable + h); 101 102 if (x->id.spi) { 103 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, 104 x->id.proto, x->props.family, 105 nhashmask); 106 hlist_add_head_rcu(&x->byspi, nspitable + h); 107 } 108 } 109 } 110 111 static unsigned long xfrm_hash_new_size(unsigned int state_hmask) 112 { 113 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head); 114 } 115 116 static void xfrm_hash_resize(struct work_struct *work) 117 { 118 struct net *net = container_of(work, struct net, xfrm.state_hash_work); 119 struct hlist_head *ndst, *nsrc, *nspi, *odst, *osrc, *ospi; 120 unsigned long nsize, osize; 121 unsigned int nhashmask, ohashmask; 122 int i; 123 124 nsize = xfrm_hash_new_size(net->xfrm.state_hmask); 125 ndst = xfrm_hash_alloc(nsize); 126 if (!ndst) 127 return; 128 nsrc = xfrm_hash_alloc(nsize); 129 if (!nsrc) { 130 xfrm_hash_free(ndst, nsize); 131 return; 132 } 133 nspi = xfrm_hash_alloc(nsize); 134 if (!nspi) { 135 xfrm_hash_free(ndst, nsize); 136 xfrm_hash_free(nsrc, nsize); 137 return; 138 } 139 140 spin_lock_bh(&net->xfrm.xfrm_state_lock); 141 write_seqcount_begin(&xfrm_state_hash_generation); 142 143 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U; 144 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net); 145 for (i = net->xfrm.state_hmask; i >= 0; i--) 146 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nhashmask); 147 148 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net); 149 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net); 150 ohashmask = net->xfrm.state_hmask; 151 152 rcu_assign_pointer(net->xfrm.state_bydst, ndst); 153 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); 154 rcu_assign_pointer(net->xfrm.state_byspi, nspi); 155 net->xfrm.state_hmask = nhashmask; 156 157 write_seqcount_end(&xfrm_state_hash_generation); 158 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 159 160 osize = (ohashmask + 1) * sizeof(struct hlist_head); 161 162 synchronize_rcu(); 163 164 xfrm_hash_free(odst, osize); 165 xfrm_hash_free(osrc, osize); 166 xfrm_hash_free(ospi, osize); 167 } 168 169 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock); 170 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO]; 171 172 static DEFINE_SPINLOCK(xfrm_state_gc_lock); 173 174 int __xfrm_state_delete(struct xfrm_state *x); 175 176 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol); 177 static bool km_is_alive(const struct km_event *c); 178 void km_state_expired(struct xfrm_state *x, int hard, u32 portid); 179 180 static DEFINE_SPINLOCK(xfrm_type_lock); 181 int xfrm_register_type(const struct xfrm_type *type, unsigned short family) 182 { 183 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 184 const struct xfrm_type **typemap; 185 int err = 0; 186 187 if (unlikely(afinfo == NULL)) 188 return -EAFNOSUPPORT; 189 typemap = afinfo->type_map; 190 spin_lock_bh(&xfrm_type_lock); 191 192 if (likely(typemap[type->proto] == NULL)) 193 typemap[type->proto] = type; 194 else 195 err = -EEXIST; 196 spin_unlock_bh(&xfrm_type_lock); 197 rcu_read_unlock(); 198 return err; 199 } 200 EXPORT_SYMBOL(xfrm_register_type); 201 202 int xfrm_unregister_type(const struct xfrm_type *type, unsigned short family) 203 { 204 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 205 const struct xfrm_type **typemap; 206 int err = 0; 207 208 if (unlikely(afinfo == NULL)) 209 return -EAFNOSUPPORT; 210 typemap = afinfo->type_map; 211 spin_lock_bh(&xfrm_type_lock); 212 213 if (unlikely(typemap[type->proto] != type)) 214 err = -ENOENT; 215 else 216 typemap[type->proto] = NULL; 217 spin_unlock_bh(&xfrm_type_lock); 218 rcu_read_unlock(); 219 return err; 220 } 221 EXPORT_SYMBOL(xfrm_unregister_type); 222 223 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family) 224 { 225 struct xfrm_state_afinfo *afinfo; 226 const struct xfrm_type **typemap; 227 const struct xfrm_type *type; 228 int modload_attempted = 0; 229 230 retry: 231 afinfo = xfrm_state_get_afinfo(family); 232 if (unlikely(afinfo == NULL)) 233 return NULL; 234 typemap = afinfo->type_map; 235 236 type = READ_ONCE(typemap[proto]); 237 if (unlikely(type && !try_module_get(type->owner))) 238 type = NULL; 239 240 rcu_read_unlock(); 241 242 if (!type && !modload_attempted) { 243 request_module("xfrm-type-%d-%d", family, proto); 244 modload_attempted = 1; 245 goto retry; 246 } 247 248 return type; 249 } 250 251 static void xfrm_put_type(const struct xfrm_type *type) 252 { 253 module_put(type->owner); 254 } 255 256 static DEFINE_SPINLOCK(xfrm_type_offload_lock); 257 int xfrm_register_type_offload(const struct xfrm_type_offload *type, 258 unsigned short family) 259 { 260 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 261 const struct xfrm_type_offload **typemap; 262 int err = 0; 263 264 if (unlikely(afinfo == NULL)) 265 return -EAFNOSUPPORT; 266 typemap = afinfo->type_offload_map; 267 spin_lock_bh(&xfrm_type_offload_lock); 268 269 if (likely(typemap[type->proto] == NULL)) 270 typemap[type->proto] = type; 271 else 272 err = -EEXIST; 273 spin_unlock_bh(&xfrm_type_offload_lock); 274 rcu_read_unlock(); 275 return err; 276 } 277 EXPORT_SYMBOL(xfrm_register_type_offload); 278 279 int xfrm_unregister_type_offload(const struct xfrm_type_offload *type, 280 unsigned short family) 281 { 282 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 283 const struct xfrm_type_offload **typemap; 284 int err = 0; 285 286 if (unlikely(afinfo == NULL)) 287 return -EAFNOSUPPORT; 288 typemap = afinfo->type_offload_map; 289 spin_lock_bh(&xfrm_type_offload_lock); 290 291 if (unlikely(typemap[type->proto] != type)) 292 err = -ENOENT; 293 else 294 typemap[type->proto] = NULL; 295 spin_unlock_bh(&xfrm_type_offload_lock); 296 rcu_read_unlock(); 297 return err; 298 } 299 EXPORT_SYMBOL(xfrm_unregister_type_offload); 300 301 static const struct xfrm_type_offload * 302 xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load) 303 { 304 struct xfrm_state_afinfo *afinfo; 305 const struct xfrm_type_offload **typemap; 306 const struct xfrm_type_offload *type; 307 308 retry: 309 afinfo = xfrm_state_get_afinfo(family); 310 if (unlikely(afinfo == NULL)) 311 return NULL; 312 typemap = afinfo->type_offload_map; 313 314 type = typemap[proto]; 315 if ((type && !try_module_get(type->owner))) 316 type = NULL; 317 318 rcu_read_unlock(); 319 320 if (!type && try_load) { 321 request_module("xfrm-offload-%d-%d", family, proto); 322 try_load = false; 323 goto retry; 324 } 325 326 return type; 327 } 328 329 static void xfrm_put_type_offload(const struct xfrm_type_offload *type) 330 { 331 module_put(type->owner); 332 } 333 334 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = { 335 [XFRM_MODE_BEET] = { 336 .encap = XFRM_MODE_BEET, 337 .flags = XFRM_MODE_FLAG_TUNNEL, 338 .family = AF_INET, 339 }, 340 [XFRM_MODE_TRANSPORT] = { 341 .encap = XFRM_MODE_TRANSPORT, 342 .family = AF_INET, 343 }, 344 [XFRM_MODE_TUNNEL] = { 345 .encap = XFRM_MODE_TUNNEL, 346 .flags = XFRM_MODE_FLAG_TUNNEL, 347 .family = AF_INET, 348 }, 349 }; 350 351 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = { 352 [XFRM_MODE_BEET] = { 353 .encap = XFRM_MODE_BEET, 354 .flags = XFRM_MODE_FLAG_TUNNEL, 355 .family = AF_INET6, 356 }, 357 [XFRM_MODE_ROUTEOPTIMIZATION] = { 358 .encap = XFRM_MODE_ROUTEOPTIMIZATION, 359 .family = AF_INET6, 360 }, 361 [XFRM_MODE_TRANSPORT] = { 362 .encap = XFRM_MODE_TRANSPORT, 363 .family = AF_INET6, 364 }, 365 [XFRM_MODE_TUNNEL] = { 366 .encap = XFRM_MODE_TUNNEL, 367 .flags = XFRM_MODE_FLAG_TUNNEL, 368 .family = AF_INET6, 369 }, 370 }; 371 372 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family) 373 { 374 const struct xfrm_mode *mode; 375 376 if (unlikely(encap >= XFRM_MODE_MAX)) 377 return NULL; 378 379 switch (family) { 380 case AF_INET: 381 mode = &xfrm4_mode_map[encap]; 382 if (mode->family == family) 383 return mode; 384 break; 385 case AF_INET6: 386 mode = &xfrm6_mode_map[encap]; 387 if (mode->family == family) 388 return mode; 389 break; 390 default: 391 break; 392 } 393 394 return NULL; 395 } 396 397 void xfrm_state_free(struct xfrm_state *x) 398 { 399 kmem_cache_free(xfrm_state_cache, x); 400 } 401 EXPORT_SYMBOL(xfrm_state_free); 402 403 static void ___xfrm_state_destroy(struct xfrm_state *x) 404 { 405 hrtimer_cancel(&x->mtimer); 406 del_timer_sync(&x->rtimer); 407 kfree(x->aead); 408 kfree(x->aalg); 409 kfree(x->ealg); 410 kfree(x->calg); 411 kfree(x->encap); 412 kfree(x->coaddr); 413 kfree(x->replay_esn); 414 kfree(x->preplay_esn); 415 if (x->type_offload) 416 xfrm_put_type_offload(x->type_offload); 417 if (x->type) { 418 x->type->destructor(x); 419 xfrm_put_type(x->type); 420 } 421 xfrm_dev_state_free(x); 422 security_xfrm_state_free(x); 423 xfrm_state_free(x); 424 } 425 426 static void xfrm_state_gc_task(struct work_struct *work) 427 { 428 struct xfrm_state *x; 429 struct hlist_node *tmp; 430 struct hlist_head gc_list; 431 432 spin_lock_bh(&xfrm_state_gc_lock); 433 hlist_move_list(&xfrm_state_gc_list, &gc_list); 434 spin_unlock_bh(&xfrm_state_gc_lock); 435 436 synchronize_rcu(); 437 438 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist) 439 ___xfrm_state_destroy(x); 440 } 441 442 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) 443 { 444 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer); 445 enum hrtimer_restart ret = HRTIMER_NORESTART; 446 time64_t now = ktime_get_real_seconds(); 447 time64_t next = TIME64_MAX; 448 int warn = 0; 449 int err = 0; 450 451 spin_lock(&x->lock); 452 if (x->km.state == XFRM_STATE_DEAD) 453 goto out; 454 if (x->km.state == XFRM_STATE_EXPIRED) 455 goto expired; 456 if (x->lft.hard_add_expires_seconds) { 457 long tmo = x->lft.hard_add_expires_seconds + 458 x->curlft.add_time - now; 459 if (tmo <= 0) { 460 if (x->xflags & XFRM_SOFT_EXPIRE) { 461 /* enter hard expire without soft expire first?! 462 * setting a new date could trigger this. 463 * workaround: fix x->curflt.add_time by below: 464 */ 465 x->curlft.add_time = now - x->saved_tmo - 1; 466 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo; 467 } else 468 goto expired; 469 } 470 if (tmo < next) 471 next = tmo; 472 } 473 if (x->lft.hard_use_expires_seconds) { 474 long tmo = x->lft.hard_use_expires_seconds + 475 (x->curlft.use_time ? : now) - now; 476 if (tmo <= 0) 477 goto expired; 478 if (tmo < next) 479 next = tmo; 480 } 481 if (x->km.dying) 482 goto resched; 483 if (x->lft.soft_add_expires_seconds) { 484 long tmo = x->lft.soft_add_expires_seconds + 485 x->curlft.add_time - now; 486 if (tmo <= 0) { 487 warn = 1; 488 x->xflags &= ~XFRM_SOFT_EXPIRE; 489 } else if (tmo < next) { 490 next = tmo; 491 x->xflags |= XFRM_SOFT_EXPIRE; 492 x->saved_tmo = tmo; 493 } 494 } 495 if (x->lft.soft_use_expires_seconds) { 496 long tmo = x->lft.soft_use_expires_seconds + 497 (x->curlft.use_time ? : now) - now; 498 if (tmo <= 0) 499 warn = 1; 500 else if (tmo < next) 501 next = tmo; 502 } 503 504 x->km.dying = warn; 505 if (warn) 506 km_state_expired(x, 0, 0); 507 resched: 508 if (next != TIME64_MAX) { 509 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0)); 510 ret = HRTIMER_RESTART; 511 } 512 513 goto out; 514 515 expired: 516 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0) 517 x->km.state = XFRM_STATE_EXPIRED; 518 519 err = __xfrm_state_delete(x); 520 if (!err) 521 km_state_expired(x, 1, 0); 522 523 xfrm_audit_state_delete(x, err ? 0 : 1, true); 524 525 out: 526 spin_unlock(&x->lock); 527 return ret; 528 } 529 530 static void xfrm_replay_timer_handler(struct timer_list *t); 531 532 struct xfrm_state *xfrm_state_alloc(struct net *net) 533 { 534 struct xfrm_state *x; 535 536 x = kmem_cache_alloc(xfrm_state_cache, GFP_ATOMIC | __GFP_ZERO); 537 538 if (x) { 539 write_pnet(&x->xs_net, net); 540 refcount_set(&x->refcnt, 1); 541 atomic_set(&x->tunnel_users, 0); 542 INIT_LIST_HEAD(&x->km.all); 543 INIT_HLIST_NODE(&x->bydst); 544 INIT_HLIST_NODE(&x->bysrc); 545 INIT_HLIST_NODE(&x->byspi); 546 hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT); 547 x->mtimer.function = xfrm_timer_handler; 548 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); 549 x->curlft.add_time = ktime_get_real_seconds(); 550 x->lft.soft_byte_limit = XFRM_INF; 551 x->lft.soft_packet_limit = XFRM_INF; 552 x->lft.hard_byte_limit = XFRM_INF; 553 x->lft.hard_packet_limit = XFRM_INF; 554 x->replay_maxage = 0; 555 x->replay_maxdiff = 0; 556 spin_lock_init(&x->lock); 557 } 558 return x; 559 } 560 EXPORT_SYMBOL(xfrm_state_alloc); 561 562 void __xfrm_state_destroy(struct xfrm_state *x, bool sync) 563 { 564 WARN_ON(x->km.state != XFRM_STATE_DEAD); 565 566 if (sync) { 567 synchronize_rcu(); 568 ___xfrm_state_destroy(x); 569 } else { 570 spin_lock_bh(&xfrm_state_gc_lock); 571 hlist_add_head(&x->gclist, &xfrm_state_gc_list); 572 spin_unlock_bh(&xfrm_state_gc_lock); 573 schedule_work(&xfrm_state_gc_work); 574 } 575 } 576 EXPORT_SYMBOL(__xfrm_state_destroy); 577 578 int __xfrm_state_delete(struct xfrm_state *x) 579 { 580 struct net *net = xs_net(x); 581 int err = -ESRCH; 582 583 if (x->km.state != XFRM_STATE_DEAD) { 584 x->km.state = XFRM_STATE_DEAD; 585 spin_lock(&net->xfrm.xfrm_state_lock); 586 list_del(&x->km.all); 587 hlist_del_rcu(&x->bydst); 588 hlist_del_rcu(&x->bysrc); 589 if (x->id.spi) 590 hlist_del_rcu(&x->byspi); 591 net->xfrm.state_num--; 592 spin_unlock(&net->xfrm.xfrm_state_lock); 593 594 xfrm_dev_state_delete(x); 595 596 /* All xfrm_state objects are created by xfrm_state_alloc. 597 * The xfrm_state_alloc call gives a reference, and that 598 * is what we are dropping here. 599 */ 600 xfrm_state_put(x); 601 err = 0; 602 } 603 604 return err; 605 } 606 EXPORT_SYMBOL(__xfrm_state_delete); 607 608 int xfrm_state_delete(struct xfrm_state *x) 609 { 610 int err; 611 612 spin_lock_bh(&x->lock); 613 err = __xfrm_state_delete(x); 614 spin_unlock_bh(&x->lock); 615 616 return err; 617 } 618 EXPORT_SYMBOL(xfrm_state_delete); 619 620 #ifdef CONFIG_SECURITY_NETWORK_XFRM 621 static inline int 622 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 623 { 624 int i, err = 0; 625 626 for (i = 0; i <= net->xfrm.state_hmask; i++) { 627 struct xfrm_state *x; 628 629 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 630 if (xfrm_id_proto_match(x->id.proto, proto) && 631 (err = security_xfrm_state_delete(x)) != 0) { 632 xfrm_audit_state_delete(x, 0, task_valid); 633 return err; 634 } 635 } 636 } 637 638 return err; 639 } 640 641 static inline int 642 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 643 { 644 int i, err = 0; 645 646 for (i = 0; i <= net->xfrm.state_hmask; i++) { 647 struct xfrm_state *x; 648 struct xfrm_state_offload *xso; 649 650 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 651 xso = &x->xso; 652 653 if (xso->dev == dev && 654 (err = security_xfrm_state_delete(x)) != 0) { 655 xfrm_audit_state_delete(x, 0, task_valid); 656 return err; 657 } 658 } 659 } 660 661 return err; 662 } 663 #else 664 static inline int 665 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 666 { 667 return 0; 668 } 669 670 static inline int 671 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 672 { 673 return 0; 674 } 675 #endif 676 677 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync) 678 { 679 int i, err = 0, cnt = 0; 680 681 spin_lock_bh(&net->xfrm.xfrm_state_lock); 682 err = xfrm_state_flush_secctx_check(net, proto, task_valid); 683 if (err) 684 goto out; 685 686 err = -ESRCH; 687 for (i = 0; i <= net->xfrm.state_hmask; i++) { 688 struct xfrm_state *x; 689 restart: 690 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 691 if (!xfrm_state_kern(x) && 692 xfrm_id_proto_match(x->id.proto, proto)) { 693 xfrm_state_hold(x); 694 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 695 696 err = xfrm_state_delete(x); 697 xfrm_audit_state_delete(x, err ? 0 : 1, 698 task_valid); 699 if (sync) 700 xfrm_state_put_sync(x); 701 else 702 xfrm_state_put(x); 703 if (!err) 704 cnt++; 705 706 spin_lock_bh(&net->xfrm.xfrm_state_lock); 707 goto restart; 708 } 709 } 710 } 711 out: 712 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 713 if (cnt) 714 err = 0; 715 716 return err; 717 } 718 EXPORT_SYMBOL(xfrm_state_flush); 719 720 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid) 721 { 722 int i, err = 0, cnt = 0; 723 724 spin_lock_bh(&net->xfrm.xfrm_state_lock); 725 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid); 726 if (err) 727 goto out; 728 729 err = -ESRCH; 730 for (i = 0; i <= net->xfrm.state_hmask; i++) { 731 struct xfrm_state *x; 732 struct xfrm_state_offload *xso; 733 restart: 734 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 735 xso = &x->xso; 736 737 if (!xfrm_state_kern(x) && xso->dev == dev) { 738 xfrm_state_hold(x); 739 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 740 741 err = xfrm_state_delete(x); 742 xfrm_audit_state_delete(x, err ? 0 : 1, 743 task_valid); 744 xfrm_state_put(x); 745 if (!err) 746 cnt++; 747 748 spin_lock_bh(&net->xfrm.xfrm_state_lock); 749 goto restart; 750 } 751 } 752 } 753 if (cnt) 754 err = 0; 755 756 out: 757 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 758 return err; 759 } 760 EXPORT_SYMBOL(xfrm_dev_state_flush); 761 762 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si) 763 { 764 spin_lock_bh(&net->xfrm.xfrm_state_lock); 765 si->sadcnt = net->xfrm.state_num; 766 si->sadhcnt = net->xfrm.state_hmask + 1; 767 si->sadhmcnt = xfrm_state_hashmax; 768 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 769 } 770 EXPORT_SYMBOL(xfrm_sad_getinfo); 771 772 static void 773 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, 774 const struct xfrm_tmpl *tmpl, 775 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 776 unsigned short family) 777 { 778 struct xfrm_state_afinfo *afinfo = xfrm_state_afinfo_get_rcu(family); 779 780 if (!afinfo) 781 return; 782 783 afinfo->init_tempsel(&x->sel, fl); 784 785 if (family != tmpl->encap_family) { 786 afinfo = xfrm_state_afinfo_get_rcu(tmpl->encap_family); 787 if (!afinfo) 788 return; 789 } 790 afinfo->init_temprop(x, tmpl, daddr, saddr); 791 } 792 793 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, 794 const xfrm_address_t *daddr, 795 __be32 spi, u8 proto, 796 unsigned short family) 797 { 798 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); 799 struct xfrm_state *x; 800 801 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { 802 if (x->props.family != family || 803 x->id.spi != spi || 804 x->id.proto != proto || 805 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 806 continue; 807 808 if ((mark & x->mark.m) != x->mark.v) 809 continue; 810 if (!xfrm_state_hold_rcu(x)) 811 continue; 812 return x; 813 } 814 815 return NULL; 816 } 817 818 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark, 819 const xfrm_address_t *daddr, 820 const xfrm_address_t *saddr, 821 u8 proto, unsigned short family) 822 { 823 unsigned int h = xfrm_src_hash(net, daddr, saddr, family); 824 struct xfrm_state *x; 825 826 hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) { 827 if (x->props.family != family || 828 x->id.proto != proto || 829 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 830 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 831 continue; 832 833 if ((mark & x->mark.m) != x->mark.v) 834 continue; 835 if (!xfrm_state_hold_rcu(x)) 836 continue; 837 return x; 838 } 839 840 return NULL; 841 } 842 843 static inline struct xfrm_state * 844 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family) 845 { 846 struct net *net = xs_net(x); 847 u32 mark = x->mark.v & x->mark.m; 848 849 if (use_spi) 850 return __xfrm_state_lookup(net, mark, &x->id.daddr, 851 x->id.spi, x->id.proto, family); 852 else 853 return __xfrm_state_lookup_byaddr(net, mark, 854 &x->id.daddr, 855 &x->props.saddr, 856 x->id.proto, family); 857 } 858 859 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision) 860 { 861 if (have_hash_collision && 862 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax && 863 net->xfrm.state_num > net->xfrm.state_hmask) 864 schedule_work(&net->xfrm.state_hash_work); 865 } 866 867 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x, 868 const struct flowi *fl, unsigned short family, 869 struct xfrm_state **best, int *acq_in_progress, 870 int *error) 871 { 872 /* Resolution logic: 873 * 1. There is a valid state with matching selector. Done. 874 * 2. Valid state with inappropriate selector. Skip. 875 * 876 * Entering area of "sysdeps". 877 * 878 * 3. If state is not valid, selector is temporary, it selects 879 * only session which triggered previous resolution. Key 880 * manager will do something to install a state with proper 881 * selector. 882 */ 883 if (x->km.state == XFRM_STATE_VALID) { 884 if ((x->sel.family && 885 !xfrm_selector_match(&x->sel, fl, x->sel.family)) || 886 !security_xfrm_state_pol_flow_match(x, pol, fl)) 887 return; 888 889 if (!*best || 890 (*best)->km.dying > x->km.dying || 891 ((*best)->km.dying == x->km.dying && 892 (*best)->curlft.add_time < x->curlft.add_time)) 893 *best = x; 894 } else if (x->km.state == XFRM_STATE_ACQ) { 895 *acq_in_progress = 1; 896 } else if (x->km.state == XFRM_STATE_ERROR || 897 x->km.state == XFRM_STATE_EXPIRED) { 898 if (xfrm_selector_match(&x->sel, fl, x->sel.family) && 899 security_xfrm_state_pol_flow_match(x, pol, fl)) 900 *error = -ESRCH; 901 } 902 } 903 904 struct xfrm_state * 905 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, 906 const struct flowi *fl, struct xfrm_tmpl *tmpl, 907 struct xfrm_policy *pol, int *err, 908 unsigned short family, u32 if_id) 909 { 910 static xfrm_address_t saddr_wildcard = { }; 911 struct net *net = xp_net(pol); 912 unsigned int h, h_wildcard; 913 struct xfrm_state *x, *x0, *to_put; 914 int acquire_in_progress = 0; 915 int error = 0; 916 struct xfrm_state *best = NULL; 917 u32 mark = pol->mark.v & pol->mark.m; 918 unsigned short encap_family = tmpl->encap_family; 919 unsigned int sequence; 920 struct km_event c; 921 922 to_put = NULL; 923 924 sequence = read_seqcount_begin(&xfrm_state_hash_generation); 925 926 rcu_read_lock(); 927 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); 928 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) { 929 if (x->props.family == encap_family && 930 x->props.reqid == tmpl->reqid && 931 (mark & x->mark.m) == x->mark.v && 932 x->if_id == if_id && 933 !(x->props.flags & XFRM_STATE_WILDRECV) && 934 xfrm_state_addr_check(x, daddr, saddr, encap_family) && 935 tmpl->mode == x->props.mode && 936 tmpl->id.proto == x->id.proto && 937 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 938 xfrm_state_look_at(pol, x, fl, encap_family, 939 &best, &acquire_in_progress, &error); 940 } 941 if (best || acquire_in_progress) 942 goto found; 943 944 h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family); 945 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) { 946 if (x->props.family == encap_family && 947 x->props.reqid == tmpl->reqid && 948 (mark & x->mark.m) == x->mark.v && 949 x->if_id == if_id && 950 !(x->props.flags & XFRM_STATE_WILDRECV) && 951 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) && 952 tmpl->mode == x->props.mode && 953 tmpl->id.proto == x->id.proto && 954 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 955 xfrm_state_look_at(pol, x, fl, encap_family, 956 &best, &acquire_in_progress, &error); 957 } 958 959 found: 960 x = best; 961 if (!x && !error && !acquire_in_progress) { 962 if (tmpl->id.spi && 963 (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi, 964 tmpl->id.proto, encap_family)) != NULL) { 965 to_put = x0; 966 error = -EEXIST; 967 goto out; 968 } 969 970 c.net = net; 971 /* If the KMs have no listeners (yet...), avoid allocating an SA 972 * for each and every packet - garbage collection might not 973 * handle the flood. 974 */ 975 if (!km_is_alive(&c)) { 976 error = -ESRCH; 977 goto out; 978 } 979 980 x = xfrm_state_alloc(net); 981 if (x == NULL) { 982 error = -ENOMEM; 983 goto out; 984 } 985 /* Initialize temporary state matching only 986 * to current session. */ 987 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family); 988 memcpy(&x->mark, &pol->mark, sizeof(x->mark)); 989 x->if_id = if_id; 990 991 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid); 992 if (error) { 993 x->km.state = XFRM_STATE_DEAD; 994 to_put = x; 995 x = NULL; 996 goto out; 997 } 998 999 if (km_query(x, tmpl, pol) == 0) { 1000 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1001 x->km.state = XFRM_STATE_ACQ; 1002 list_add(&x->km.all, &net->xfrm.state_all); 1003 hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); 1004 h = xfrm_src_hash(net, daddr, saddr, encap_family); 1005 hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); 1006 if (x->id.spi) { 1007 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); 1008 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); 1009 } 1010 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1011 hrtimer_start(&x->mtimer, 1012 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1013 HRTIMER_MODE_REL_SOFT); 1014 net->xfrm.state_num++; 1015 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1016 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1017 } else { 1018 x->km.state = XFRM_STATE_DEAD; 1019 to_put = x; 1020 x = NULL; 1021 error = -ESRCH; 1022 } 1023 } 1024 out: 1025 if (x) { 1026 if (!xfrm_state_hold_rcu(x)) { 1027 *err = -EAGAIN; 1028 x = NULL; 1029 } 1030 } else { 1031 *err = acquire_in_progress ? -EAGAIN : error; 1032 } 1033 rcu_read_unlock(); 1034 if (to_put) 1035 xfrm_state_put(to_put); 1036 1037 if (read_seqcount_retry(&xfrm_state_hash_generation, sequence)) { 1038 *err = -EAGAIN; 1039 if (x) { 1040 xfrm_state_put(x); 1041 x = NULL; 1042 } 1043 } 1044 1045 return x; 1046 } 1047 1048 struct xfrm_state * 1049 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, 1050 xfrm_address_t *daddr, xfrm_address_t *saddr, 1051 unsigned short family, u8 mode, u8 proto, u32 reqid) 1052 { 1053 unsigned int h; 1054 struct xfrm_state *rx = NULL, *x = NULL; 1055 1056 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1057 h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1058 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1059 if (x->props.family == family && 1060 x->props.reqid == reqid && 1061 (mark & x->mark.m) == x->mark.v && 1062 x->if_id == if_id && 1063 !(x->props.flags & XFRM_STATE_WILDRECV) && 1064 xfrm_state_addr_check(x, daddr, saddr, family) && 1065 mode == x->props.mode && 1066 proto == x->id.proto && 1067 x->km.state == XFRM_STATE_VALID) { 1068 rx = x; 1069 break; 1070 } 1071 } 1072 1073 if (rx) 1074 xfrm_state_hold(rx); 1075 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1076 1077 1078 return rx; 1079 } 1080 EXPORT_SYMBOL(xfrm_stateonly_find); 1081 1082 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, 1083 unsigned short family) 1084 { 1085 struct xfrm_state *x; 1086 struct xfrm_state_walk *w; 1087 1088 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1089 list_for_each_entry(w, &net->xfrm.state_all, all) { 1090 x = container_of(w, struct xfrm_state, km); 1091 if (x->props.family != family || 1092 x->id.spi != spi) 1093 continue; 1094 1095 xfrm_state_hold(x); 1096 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1097 return x; 1098 } 1099 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1100 return NULL; 1101 } 1102 EXPORT_SYMBOL(xfrm_state_lookup_byspi); 1103 1104 static void __xfrm_state_insert(struct xfrm_state *x) 1105 { 1106 struct net *net = xs_net(x); 1107 unsigned int h; 1108 1109 list_add(&x->km.all, &net->xfrm.state_all); 1110 1111 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, 1112 x->props.reqid, x->props.family); 1113 hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); 1114 1115 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); 1116 hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); 1117 1118 if (x->id.spi) { 1119 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, 1120 x->props.family); 1121 1122 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); 1123 } 1124 1125 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); 1126 if (x->replay_maxage) 1127 mod_timer(&x->rtimer, jiffies + x->replay_maxage); 1128 1129 net->xfrm.state_num++; 1130 1131 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1132 } 1133 1134 /* net->xfrm.xfrm_state_lock is held */ 1135 static void __xfrm_state_bump_genids(struct xfrm_state *xnew) 1136 { 1137 struct net *net = xs_net(xnew); 1138 unsigned short family = xnew->props.family; 1139 u32 reqid = xnew->props.reqid; 1140 struct xfrm_state *x; 1141 unsigned int h; 1142 u32 mark = xnew->mark.v & xnew->mark.m; 1143 u32 if_id = xnew->if_id; 1144 1145 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); 1146 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1147 if (x->props.family == family && 1148 x->props.reqid == reqid && 1149 x->if_id == if_id && 1150 (mark & x->mark.m) == x->mark.v && 1151 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) && 1152 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) 1153 x->genid++; 1154 } 1155 } 1156 1157 void xfrm_state_insert(struct xfrm_state *x) 1158 { 1159 struct net *net = xs_net(x); 1160 1161 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1162 __xfrm_state_bump_genids(x); 1163 __xfrm_state_insert(x); 1164 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1165 } 1166 EXPORT_SYMBOL(xfrm_state_insert); 1167 1168 /* net->xfrm.xfrm_state_lock is held */ 1169 static struct xfrm_state *__find_acq_core(struct net *net, 1170 const struct xfrm_mark *m, 1171 unsigned short family, u8 mode, 1172 u32 reqid, u32 if_id, u8 proto, 1173 const xfrm_address_t *daddr, 1174 const xfrm_address_t *saddr, 1175 int create) 1176 { 1177 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1178 struct xfrm_state *x; 1179 u32 mark = m->v & m->m; 1180 1181 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1182 if (x->props.reqid != reqid || 1183 x->props.mode != mode || 1184 x->props.family != family || 1185 x->km.state != XFRM_STATE_ACQ || 1186 x->id.spi != 0 || 1187 x->id.proto != proto || 1188 (mark & x->mark.m) != x->mark.v || 1189 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1190 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1191 continue; 1192 1193 xfrm_state_hold(x); 1194 return x; 1195 } 1196 1197 if (!create) 1198 return NULL; 1199 1200 x = xfrm_state_alloc(net); 1201 if (likely(x)) { 1202 switch (family) { 1203 case AF_INET: 1204 x->sel.daddr.a4 = daddr->a4; 1205 x->sel.saddr.a4 = saddr->a4; 1206 x->sel.prefixlen_d = 32; 1207 x->sel.prefixlen_s = 32; 1208 x->props.saddr.a4 = saddr->a4; 1209 x->id.daddr.a4 = daddr->a4; 1210 break; 1211 1212 case AF_INET6: 1213 x->sel.daddr.in6 = daddr->in6; 1214 x->sel.saddr.in6 = saddr->in6; 1215 x->sel.prefixlen_d = 128; 1216 x->sel.prefixlen_s = 128; 1217 x->props.saddr.in6 = saddr->in6; 1218 x->id.daddr.in6 = daddr->in6; 1219 break; 1220 } 1221 1222 x->km.state = XFRM_STATE_ACQ; 1223 x->id.proto = proto; 1224 x->props.family = family; 1225 x->props.mode = mode; 1226 x->props.reqid = reqid; 1227 x->if_id = if_id; 1228 x->mark.v = m->v; 1229 x->mark.m = m->m; 1230 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1231 xfrm_state_hold(x); 1232 hrtimer_start(&x->mtimer, 1233 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1234 HRTIMER_MODE_REL_SOFT); 1235 list_add(&x->km.all, &net->xfrm.state_all); 1236 hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h); 1237 h = xfrm_src_hash(net, daddr, saddr, family); 1238 hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h); 1239 1240 net->xfrm.state_num++; 1241 1242 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1243 } 1244 1245 return x; 1246 } 1247 1248 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); 1249 1250 int xfrm_state_add(struct xfrm_state *x) 1251 { 1252 struct net *net = xs_net(x); 1253 struct xfrm_state *x1, *to_put; 1254 int family; 1255 int err; 1256 u32 mark = x->mark.v & x->mark.m; 1257 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1258 1259 family = x->props.family; 1260 1261 to_put = NULL; 1262 1263 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1264 1265 x1 = __xfrm_state_locate(x, use_spi, family); 1266 if (x1) { 1267 to_put = x1; 1268 x1 = NULL; 1269 err = -EEXIST; 1270 goto out; 1271 } 1272 1273 if (use_spi && x->km.seq) { 1274 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq); 1275 if (x1 && ((x1->id.proto != x->id.proto) || 1276 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) { 1277 to_put = x1; 1278 x1 = NULL; 1279 } 1280 } 1281 1282 if (use_spi && !x1) 1283 x1 = __find_acq_core(net, &x->mark, family, x->props.mode, 1284 x->props.reqid, x->if_id, x->id.proto, 1285 &x->id.daddr, &x->props.saddr, 0); 1286 1287 __xfrm_state_bump_genids(x); 1288 __xfrm_state_insert(x); 1289 err = 0; 1290 1291 out: 1292 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1293 1294 if (x1) { 1295 xfrm_state_delete(x1); 1296 xfrm_state_put(x1); 1297 } 1298 1299 if (to_put) 1300 xfrm_state_put(to_put); 1301 1302 return err; 1303 } 1304 EXPORT_SYMBOL(xfrm_state_add); 1305 1306 #ifdef CONFIG_XFRM_MIGRATE 1307 static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig, 1308 struct xfrm_encap_tmpl *encap) 1309 { 1310 struct net *net = xs_net(orig); 1311 struct xfrm_state *x = xfrm_state_alloc(net); 1312 if (!x) 1313 goto out; 1314 1315 memcpy(&x->id, &orig->id, sizeof(x->id)); 1316 memcpy(&x->sel, &orig->sel, sizeof(x->sel)); 1317 memcpy(&x->lft, &orig->lft, sizeof(x->lft)); 1318 x->props.mode = orig->props.mode; 1319 x->props.replay_window = orig->props.replay_window; 1320 x->props.reqid = orig->props.reqid; 1321 x->props.family = orig->props.family; 1322 x->props.saddr = orig->props.saddr; 1323 1324 if (orig->aalg) { 1325 x->aalg = xfrm_algo_auth_clone(orig->aalg); 1326 if (!x->aalg) 1327 goto error; 1328 } 1329 x->props.aalgo = orig->props.aalgo; 1330 1331 if (orig->aead) { 1332 x->aead = xfrm_algo_aead_clone(orig->aead); 1333 x->geniv = orig->geniv; 1334 if (!x->aead) 1335 goto error; 1336 } 1337 if (orig->ealg) { 1338 x->ealg = xfrm_algo_clone(orig->ealg); 1339 if (!x->ealg) 1340 goto error; 1341 } 1342 x->props.ealgo = orig->props.ealgo; 1343 1344 if (orig->calg) { 1345 x->calg = xfrm_algo_clone(orig->calg); 1346 if (!x->calg) 1347 goto error; 1348 } 1349 x->props.calgo = orig->props.calgo; 1350 1351 if (encap || orig->encap) { 1352 if (encap) 1353 x->encap = kmemdup(encap, sizeof(*x->encap), 1354 GFP_KERNEL); 1355 else 1356 x->encap = kmemdup(orig->encap, sizeof(*x->encap), 1357 GFP_KERNEL); 1358 1359 if (!x->encap) 1360 goto error; 1361 } 1362 1363 if (orig->coaddr) { 1364 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr), 1365 GFP_KERNEL); 1366 if (!x->coaddr) 1367 goto error; 1368 } 1369 1370 if (orig->replay_esn) { 1371 if (xfrm_replay_clone(x, orig)) 1372 goto error; 1373 } 1374 1375 memcpy(&x->mark, &orig->mark, sizeof(x->mark)); 1376 1377 if (xfrm_init_state(x) < 0) 1378 goto error; 1379 1380 x->props.flags = orig->props.flags; 1381 x->props.extra_flags = orig->props.extra_flags; 1382 1383 x->if_id = orig->if_id; 1384 x->tfcpad = orig->tfcpad; 1385 x->replay_maxdiff = orig->replay_maxdiff; 1386 x->replay_maxage = orig->replay_maxage; 1387 x->curlft.add_time = orig->curlft.add_time; 1388 x->km.state = orig->km.state; 1389 x->km.seq = orig->km.seq; 1390 x->replay = orig->replay; 1391 x->preplay = orig->preplay; 1392 1393 return x; 1394 1395 error: 1396 xfrm_state_put(x); 1397 out: 1398 return NULL; 1399 } 1400 1401 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net) 1402 { 1403 unsigned int h; 1404 struct xfrm_state *x = NULL; 1405 1406 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1407 1408 if (m->reqid) { 1409 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr, 1410 m->reqid, m->old_family); 1411 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1412 if (x->props.mode != m->mode || 1413 x->id.proto != m->proto) 1414 continue; 1415 if (m->reqid && x->props.reqid != m->reqid) 1416 continue; 1417 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1418 m->old_family) || 1419 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1420 m->old_family)) 1421 continue; 1422 xfrm_state_hold(x); 1423 break; 1424 } 1425 } else { 1426 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr, 1427 m->old_family); 1428 hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) { 1429 if (x->props.mode != m->mode || 1430 x->id.proto != m->proto) 1431 continue; 1432 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1433 m->old_family) || 1434 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1435 m->old_family)) 1436 continue; 1437 xfrm_state_hold(x); 1438 break; 1439 } 1440 } 1441 1442 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1443 1444 return x; 1445 } 1446 EXPORT_SYMBOL(xfrm_migrate_state_find); 1447 1448 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x, 1449 struct xfrm_migrate *m, 1450 struct xfrm_encap_tmpl *encap) 1451 { 1452 struct xfrm_state *xc; 1453 1454 xc = xfrm_state_clone(x, encap); 1455 if (!xc) 1456 return NULL; 1457 1458 memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr)); 1459 memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr)); 1460 1461 /* add state */ 1462 if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) { 1463 /* a care is needed when the destination address of the 1464 state is to be updated as it is a part of triplet */ 1465 xfrm_state_insert(xc); 1466 } else { 1467 if (xfrm_state_add(xc) < 0) 1468 goto error; 1469 } 1470 1471 return xc; 1472 error: 1473 xfrm_state_put(xc); 1474 return NULL; 1475 } 1476 EXPORT_SYMBOL(xfrm_state_migrate); 1477 #endif 1478 1479 int xfrm_state_update(struct xfrm_state *x) 1480 { 1481 struct xfrm_state *x1, *to_put; 1482 int err; 1483 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1484 struct net *net = xs_net(x); 1485 1486 to_put = NULL; 1487 1488 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1489 x1 = __xfrm_state_locate(x, use_spi, x->props.family); 1490 1491 err = -ESRCH; 1492 if (!x1) 1493 goto out; 1494 1495 if (xfrm_state_kern(x1)) { 1496 to_put = x1; 1497 err = -EEXIST; 1498 goto out; 1499 } 1500 1501 if (x1->km.state == XFRM_STATE_ACQ) { 1502 __xfrm_state_insert(x); 1503 x = NULL; 1504 } 1505 err = 0; 1506 1507 out: 1508 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1509 1510 if (to_put) 1511 xfrm_state_put(to_put); 1512 1513 if (err) 1514 return err; 1515 1516 if (!x) { 1517 xfrm_state_delete(x1); 1518 xfrm_state_put(x1); 1519 return 0; 1520 } 1521 1522 err = -EINVAL; 1523 spin_lock_bh(&x1->lock); 1524 if (likely(x1->km.state == XFRM_STATE_VALID)) { 1525 if (x->encap && x1->encap && 1526 x->encap->encap_type == x1->encap->encap_type) 1527 memcpy(x1->encap, x->encap, sizeof(*x1->encap)); 1528 else if (x->encap || x1->encap) 1529 goto fail; 1530 1531 if (x->coaddr && x1->coaddr) { 1532 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr)); 1533 } 1534 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel))) 1535 memcpy(&x1->sel, &x->sel, sizeof(x1->sel)); 1536 memcpy(&x1->lft, &x->lft, sizeof(x1->lft)); 1537 x1->km.dying = 0; 1538 1539 hrtimer_start(&x1->mtimer, ktime_set(1, 0), 1540 HRTIMER_MODE_REL_SOFT); 1541 if (x1->curlft.use_time) 1542 xfrm_state_check_expire(x1); 1543 1544 if (x->props.smark.m || x->props.smark.v || x->if_id) { 1545 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1546 1547 if (x->props.smark.m || x->props.smark.v) 1548 x1->props.smark = x->props.smark; 1549 1550 if (x->if_id) 1551 x1->if_id = x->if_id; 1552 1553 __xfrm_state_bump_genids(x1); 1554 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1555 } 1556 1557 err = 0; 1558 x->km.state = XFRM_STATE_DEAD; 1559 __xfrm_state_put(x); 1560 } 1561 1562 fail: 1563 spin_unlock_bh(&x1->lock); 1564 1565 xfrm_state_put(x1); 1566 1567 return err; 1568 } 1569 EXPORT_SYMBOL(xfrm_state_update); 1570 1571 int xfrm_state_check_expire(struct xfrm_state *x) 1572 { 1573 if (!x->curlft.use_time) 1574 x->curlft.use_time = ktime_get_real_seconds(); 1575 1576 if (x->curlft.bytes >= x->lft.hard_byte_limit || 1577 x->curlft.packets >= x->lft.hard_packet_limit) { 1578 x->km.state = XFRM_STATE_EXPIRED; 1579 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT); 1580 return -EINVAL; 1581 } 1582 1583 if (!x->km.dying && 1584 (x->curlft.bytes >= x->lft.soft_byte_limit || 1585 x->curlft.packets >= x->lft.soft_packet_limit)) { 1586 x->km.dying = 1; 1587 km_state_expired(x, 0, 0); 1588 } 1589 return 0; 1590 } 1591 EXPORT_SYMBOL(xfrm_state_check_expire); 1592 1593 struct xfrm_state * 1594 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, 1595 u8 proto, unsigned short family) 1596 { 1597 struct xfrm_state *x; 1598 1599 rcu_read_lock(); 1600 x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family); 1601 rcu_read_unlock(); 1602 return x; 1603 } 1604 EXPORT_SYMBOL(xfrm_state_lookup); 1605 1606 struct xfrm_state * 1607 xfrm_state_lookup_byaddr(struct net *net, u32 mark, 1608 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1609 u8 proto, unsigned short family) 1610 { 1611 struct xfrm_state *x; 1612 1613 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1614 x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family); 1615 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1616 return x; 1617 } 1618 EXPORT_SYMBOL(xfrm_state_lookup_byaddr); 1619 1620 struct xfrm_state * 1621 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid, 1622 u32 if_id, u8 proto, const xfrm_address_t *daddr, 1623 const xfrm_address_t *saddr, int create, unsigned short family) 1624 { 1625 struct xfrm_state *x; 1626 1627 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1628 x = __find_acq_core(net, mark, family, mode, reqid, if_id, proto, daddr, saddr, create); 1629 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1630 1631 return x; 1632 } 1633 EXPORT_SYMBOL(xfrm_find_acq); 1634 1635 #ifdef CONFIG_XFRM_SUB_POLICY 1636 int 1637 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n, 1638 unsigned short family, struct net *net) 1639 { 1640 int i; 1641 int err = 0; 1642 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 1643 if (!afinfo) 1644 return -EAFNOSUPPORT; 1645 1646 spin_lock_bh(&net->xfrm.xfrm_state_lock); /*FIXME*/ 1647 if (afinfo->tmpl_sort) 1648 err = afinfo->tmpl_sort(dst, src, n); 1649 else 1650 for (i = 0; i < n; i++) 1651 dst[i] = src[i]; 1652 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1653 rcu_read_unlock(); 1654 return err; 1655 } 1656 EXPORT_SYMBOL(xfrm_tmpl_sort); 1657 1658 int 1659 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n, 1660 unsigned short family) 1661 { 1662 int i; 1663 int err = 0; 1664 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 1665 struct net *net = xs_net(*src); 1666 1667 if (!afinfo) 1668 return -EAFNOSUPPORT; 1669 1670 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1671 if (afinfo->state_sort) 1672 err = afinfo->state_sort(dst, src, n); 1673 else 1674 for (i = 0; i < n; i++) 1675 dst[i] = src[i]; 1676 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1677 rcu_read_unlock(); 1678 return err; 1679 } 1680 EXPORT_SYMBOL(xfrm_state_sort); 1681 #endif 1682 1683 /* Silly enough, but I'm lazy to build resolution list */ 1684 1685 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 1686 { 1687 int i; 1688 1689 for (i = 0; i <= net->xfrm.state_hmask; i++) { 1690 struct xfrm_state *x; 1691 1692 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 1693 if (x->km.seq == seq && 1694 (mark & x->mark.m) == x->mark.v && 1695 x->km.state == XFRM_STATE_ACQ) { 1696 xfrm_state_hold(x); 1697 return x; 1698 } 1699 } 1700 } 1701 return NULL; 1702 } 1703 1704 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 1705 { 1706 struct xfrm_state *x; 1707 1708 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1709 x = __xfrm_find_acq_byseq(net, mark, seq); 1710 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1711 return x; 1712 } 1713 EXPORT_SYMBOL(xfrm_find_acq_byseq); 1714 1715 u32 xfrm_get_acqseq(void) 1716 { 1717 u32 res; 1718 static atomic_t acqseq; 1719 1720 do { 1721 res = atomic_inc_return(&acqseq); 1722 } while (!res); 1723 1724 return res; 1725 } 1726 EXPORT_SYMBOL(xfrm_get_acqseq); 1727 1728 int verify_spi_info(u8 proto, u32 min, u32 max) 1729 { 1730 switch (proto) { 1731 case IPPROTO_AH: 1732 case IPPROTO_ESP: 1733 break; 1734 1735 case IPPROTO_COMP: 1736 /* IPCOMP spi is 16-bits. */ 1737 if (max >= 0x10000) 1738 return -EINVAL; 1739 break; 1740 1741 default: 1742 return -EINVAL; 1743 } 1744 1745 if (min > max) 1746 return -EINVAL; 1747 1748 return 0; 1749 } 1750 EXPORT_SYMBOL(verify_spi_info); 1751 1752 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high) 1753 { 1754 struct net *net = xs_net(x); 1755 unsigned int h; 1756 struct xfrm_state *x0; 1757 int err = -ENOENT; 1758 __be32 minspi = htonl(low); 1759 __be32 maxspi = htonl(high); 1760 u32 mark = x->mark.v & x->mark.m; 1761 1762 spin_lock_bh(&x->lock); 1763 if (x->km.state == XFRM_STATE_DEAD) 1764 goto unlock; 1765 1766 err = 0; 1767 if (x->id.spi) 1768 goto unlock; 1769 1770 err = -ENOENT; 1771 1772 if (minspi == maxspi) { 1773 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family); 1774 if (x0) { 1775 xfrm_state_put(x0); 1776 goto unlock; 1777 } 1778 x->id.spi = minspi; 1779 } else { 1780 u32 spi = 0; 1781 for (h = 0; h < high-low+1; h++) { 1782 spi = low + prandom_u32()%(high-low+1); 1783 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family); 1784 if (x0 == NULL) { 1785 x->id.spi = htonl(spi); 1786 break; 1787 } 1788 xfrm_state_put(x0); 1789 } 1790 } 1791 if (x->id.spi) { 1792 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1793 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); 1794 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h); 1795 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1796 1797 err = 0; 1798 } 1799 1800 unlock: 1801 spin_unlock_bh(&x->lock); 1802 1803 return err; 1804 } 1805 EXPORT_SYMBOL(xfrm_alloc_spi); 1806 1807 static bool __xfrm_state_filter_match(struct xfrm_state *x, 1808 struct xfrm_address_filter *filter) 1809 { 1810 if (filter) { 1811 if ((filter->family == AF_INET || 1812 filter->family == AF_INET6) && 1813 x->props.family != filter->family) 1814 return false; 1815 1816 return addr_match(&x->props.saddr, &filter->saddr, 1817 filter->splen) && 1818 addr_match(&x->id.daddr, &filter->daddr, 1819 filter->dplen); 1820 } 1821 return true; 1822 } 1823 1824 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk, 1825 int (*func)(struct xfrm_state *, int, void*), 1826 void *data) 1827 { 1828 struct xfrm_state *state; 1829 struct xfrm_state_walk *x; 1830 int err = 0; 1831 1832 if (walk->seq != 0 && list_empty(&walk->all)) 1833 return 0; 1834 1835 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1836 if (list_empty(&walk->all)) 1837 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all); 1838 else 1839 x = list_first_entry(&walk->all, struct xfrm_state_walk, all); 1840 list_for_each_entry_from(x, &net->xfrm.state_all, all) { 1841 if (x->state == XFRM_STATE_DEAD) 1842 continue; 1843 state = container_of(x, struct xfrm_state, km); 1844 if (!xfrm_id_proto_match(state->id.proto, walk->proto)) 1845 continue; 1846 if (!__xfrm_state_filter_match(state, walk->filter)) 1847 continue; 1848 err = func(state, walk->seq, data); 1849 if (err) { 1850 list_move_tail(&walk->all, &x->all); 1851 goto out; 1852 } 1853 walk->seq++; 1854 } 1855 if (walk->seq == 0) { 1856 err = -ENOENT; 1857 goto out; 1858 } 1859 list_del_init(&walk->all); 1860 out: 1861 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1862 return err; 1863 } 1864 EXPORT_SYMBOL(xfrm_state_walk); 1865 1866 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto, 1867 struct xfrm_address_filter *filter) 1868 { 1869 INIT_LIST_HEAD(&walk->all); 1870 walk->proto = proto; 1871 walk->state = XFRM_STATE_DEAD; 1872 walk->seq = 0; 1873 walk->filter = filter; 1874 } 1875 EXPORT_SYMBOL(xfrm_state_walk_init); 1876 1877 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net) 1878 { 1879 kfree(walk->filter); 1880 1881 if (list_empty(&walk->all)) 1882 return; 1883 1884 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1885 list_del(&walk->all); 1886 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1887 } 1888 EXPORT_SYMBOL(xfrm_state_walk_done); 1889 1890 static void xfrm_replay_timer_handler(struct timer_list *t) 1891 { 1892 struct xfrm_state *x = from_timer(x, t, rtimer); 1893 1894 spin_lock(&x->lock); 1895 1896 if (x->km.state == XFRM_STATE_VALID) { 1897 if (xfrm_aevent_is_on(xs_net(x))) 1898 x->repl->notify(x, XFRM_REPLAY_TIMEOUT); 1899 else 1900 x->xflags |= XFRM_TIME_DEFER; 1901 } 1902 1903 spin_unlock(&x->lock); 1904 } 1905 1906 static LIST_HEAD(xfrm_km_list); 1907 1908 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c) 1909 { 1910 struct xfrm_mgr *km; 1911 1912 rcu_read_lock(); 1913 list_for_each_entry_rcu(km, &xfrm_km_list, list) 1914 if (km->notify_policy) 1915 km->notify_policy(xp, dir, c); 1916 rcu_read_unlock(); 1917 } 1918 1919 void km_state_notify(struct xfrm_state *x, const struct km_event *c) 1920 { 1921 struct xfrm_mgr *km; 1922 rcu_read_lock(); 1923 list_for_each_entry_rcu(km, &xfrm_km_list, list) 1924 if (km->notify) 1925 km->notify(x, c); 1926 rcu_read_unlock(); 1927 } 1928 1929 EXPORT_SYMBOL(km_policy_notify); 1930 EXPORT_SYMBOL(km_state_notify); 1931 1932 void km_state_expired(struct xfrm_state *x, int hard, u32 portid) 1933 { 1934 struct km_event c; 1935 1936 c.data.hard = hard; 1937 c.portid = portid; 1938 c.event = XFRM_MSG_EXPIRE; 1939 km_state_notify(x, &c); 1940 } 1941 1942 EXPORT_SYMBOL(km_state_expired); 1943 /* 1944 * We send to all registered managers regardless of failure 1945 * We are happy with one success 1946 */ 1947 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol) 1948 { 1949 int err = -EINVAL, acqret; 1950 struct xfrm_mgr *km; 1951 1952 rcu_read_lock(); 1953 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 1954 acqret = km->acquire(x, t, pol); 1955 if (!acqret) 1956 err = acqret; 1957 } 1958 rcu_read_unlock(); 1959 return err; 1960 } 1961 EXPORT_SYMBOL(km_query); 1962 1963 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 1964 { 1965 int err = -EINVAL; 1966 struct xfrm_mgr *km; 1967 1968 rcu_read_lock(); 1969 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 1970 if (km->new_mapping) 1971 err = km->new_mapping(x, ipaddr, sport); 1972 if (!err) 1973 break; 1974 } 1975 rcu_read_unlock(); 1976 return err; 1977 } 1978 EXPORT_SYMBOL(km_new_mapping); 1979 1980 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid) 1981 { 1982 struct km_event c; 1983 1984 c.data.hard = hard; 1985 c.portid = portid; 1986 c.event = XFRM_MSG_POLEXPIRE; 1987 km_policy_notify(pol, dir, &c); 1988 } 1989 EXPORT_SYMBOL(km_policy_expired); 1990 1991 #ifdef CONFIG_XFRM_MIGRATE 1992 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, 1993 const struct xfrm_migrate *m, int num_migrate, 1994 const struct xfrm_kmaddress *k, 1995 const struct xfrm_encap_tmpl *encap) 1996 { 1997 int err = -EINVAL; 1998 int ret; 1999 struct xfrm_mgr *km; 2000 2001 rcu_read_lock(); 2002 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2003 if (km->migrate) { 2004 ret = km->migrate(sel, dir, type, m, num_migrate, k, 2005 encap); 2006 if (!ret) 2007 err = ret; 2008 } 2009 } 2010 rcu_read_unlock(); 2011 return err; 2012 } 2013 EXPORT_SYMBOL(km_migrate); 2014 #endif 2015 2016 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr) 2017 { 2018 int err = -EINVAL; 2019 int ret; 2020 struct xfrm_mgr *km; 2021 2022 rcu_read_lock(); 2023 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2024 if (km->report) { 2025 ret = km->report(net, proto, sel, addr); 2026 if (!ret) 2027 err = ret; 2028 } 2029 } 2030 rcu_read_unlock(); 2031 return err; 2032 } 2033 EXPORT_SYMBOL(km_report); 2034 2035 static bool km_is_alive(const struct km_event *c) 2036 { 2037 struct xfrm_mgr *km; 2038 bool is_alive = false; 2039 2040 rcu_read_lock(); 2041 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2042 if (km->is_alive && km->is_alive(c)) { 2043 is_alive = true; 2044 break; 2045 } 2046 } 2047 rcu_read_unlock(); 2048 2049 return is_alive; 2050 } 2051 2052 int xfrm_user_policy(struct sock *sk, int optname, u8 __user *optval, int optlen) 2053 { 2054 int err; 2055 u8 *data; 2056 struct xfrm_mgr *km; 2057 struct xfrm_policy *pol = NULL; 2058 2059 if (in_compat_syscall()) 2060 return -EOPNOTSUPP; 2061 2062 if (!optval && !optlen) { 2063 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); 2064 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); 2065 __sk_dst_reset(sk); 2066 return 0; 2067 } 2068 2069 if (optlen <= 0 || optlen > PAGE_SIZE) 2070 return -EMSGSIZE; 2071 2072 data = memdup_user(optval, optlen); 2073 if (IS_ERR(data)) 2074 return PTR_ERR(data); 2075 2076 err = -EINVAL; 2077 rcu_read_lock(); 2078 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2079 pol = km->compile_policy(sk, optname, data, 2080 optlen, &err); 2081 if (err >= 0) 2082 break; 2083 } 2084 rcu_read_unlock(); 2085 2086 if (err >= 0) { 2087 xfrm_sk_policy_insert(sk, err, pol); 2088 xfrm_pol_put(pol); 2089 __sk_dst_reset(sk); 2090 err = 0; 2091 } 2092 2093 kfree(data); 2094 return err; 2095 } 2096 EXPORT_SYMBOL(xfrm_user_policy); 2097 2098 static DEFINE_SPINLOCK(xfrm_km_lock); 2099 2100 int xfrm_register_km(struct xfrm_mgr *km) 2101 { 2102 spin_lock_bh(&xfrm_km_lock); 2103 list_add_tail_rcu(&km->list, &xfrm_km_list); 2104 spin_unlock_bh(&xfrm_km_lock); 2105 return 0; 2106 } 2107 EXPORT_SYMBOL(xfrm_register_km); 2108 2109 int xfrm_unregister_km(struct xfrm_mgr *km) 2110 { 2111 spin_lock_bh(&xfrm_km_lock); 2112 list_del_rcu(&km->list); 2113 spin_unlock_bh(&xfrm_km_lock); 2114 synchronize_rcu(); 2115 return 0; 2116 } 2117 EXPORT_SYMBOL(xfrm_unregister_km); 2118 2119 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo) 2120 { 2121 int err = 0; 2122 2123 if (WARN_ON(afinfo->family >= NPROTO)) 2124 return -EAFNOSUPPORT; 2125 2126 spin_lock_bh(&xfrm_state_afinfo_lock); 2127 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL)) 2128 err = -EEXIST; 2129 else 2130 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo); 2131 spin_unlock_bh(&xfrm_state_afinfo_lock); 2132 return err; 2133 } 2134 EXPORT_SYMBOL(xfrm_state_register_afinfo); 2135 2136 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo) 2137 { 2138 int err = 0, family = afinfo->family; 2139 2140 if (WARN_ON(family >= NPROTO)) 2141 return -EAFNOSUPPORT; 2142 2143 spin_lock_bh(&xfrm_state_afinfo_lock); 2144 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) { 2145 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo) 2146 err = -EINVAL; 2147 else 2148 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL); 2149 } 2150 spin_unlock_bh(&xfrm_state_afinfo_lock); 2151 synchronize_rcu(); 2152 return err; 2153 } 2154 EXPORT_SYMBOL(xfrm_state_unregister_afinfo); 2155 2156 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family) 2157 { 2158 if (unlikely(family >= NPROTO)) 2159 return NULL; 2160 2161 return rcu_dereference(xfrm_state_afinfo[family]); 2162 } 2163 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu); 2164 2165 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family) 2166 { 2167 struct xfrm_state_afinfo *afinfo; 2168 if (unlikely(family >= NPROTO)) 2169 return NULL; 2170 rcu_read_lock(); 2171 afinfo = rcu_dereference(xfrm_state_afinfo[family]); 2172 if (unlikely(!afinfo)) 2173 rcu_read_unlock(); 2174 return afinfo; 2175 } 2176 2177 void xfrm_flush_gc(void) 2178 { 2179 flush_work(&xfrm_state_gc_work); 2180 } 2181 EXPORT_SYMBOL(xfrm_flush_gc); 2182 2183 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */ 2184 void xfrm_state_delete_tunnel(struct xfrm_state *x) 2185 { 2186 if (x->tunnel) { 2187 struct xfrm_state *t = x->tunnel; 2188 2189 if (atomic_read(&t->tunnel_users) == 2) 2190 xfrm_state_delete(t); 2191 atomic_dec(&t->tunnel_users); 2192 xfrm_state_put_sync(t); 2193 x->tunnel = NULL; 2194 } 2195 } 2196 EXPORT_SYMBOL(xfrm_state_delete_tunnel); 2197 2198 int xfrm_state_mtu(struct xfrm_state *x, int mtu) 2199 { 2200 const struct xfrm_type *type = READ_ONCE(x->type); 2201 2202 if (x->km.state == XFRM_STATE_VALID && 2203 type && type->get_mtu) 2204 return type->get_mtu(x, mtu); 2205 2206 return mtu - x->props.header_len; 2207 } 2208 2209 int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) 2210 { 2211 const struct xfrm_state_afinfo *afinfo; 2212 const struct xfrm_mode *inner_mode; 2213 const struct xfrm_mode *outer_mode; 2214 int family = x->props.family; 2215 int err; 2216 2217 err = -EAFNOSUPPORT; 2218 afinfo = xfrm_state_get_afinfo(family); 2219 if (!afinfo) 2220 goto error; 2221 2222 err = 0; 2223 if (afinfo->init_flags) 2224 err = afinfo->init_flags(x); 2225 2226 rcu_read_unlock(); 2227 2228 if (err) 2229 goto error; 2230 2231 err = -EPROTONOSUPPORT; 2232 2233 if (x->sel.family != AF_UNSPEC) { 2234 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family); 2235 if (inner_mode == NULL) 2236 goto error; 2237 2238 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) && 2239 family != x->sel.family) 2240 goto error; 2241 2242 x->inner_mode = *inner_mode; 2243 } else { 2244 const struct xfrm_mode *inner_mode_iaf; 2245 int iafamily = AF_INET; 2246 2247 inner_mode = xfrm_get_mode(x->props.mode, x->props.family); 2248 if (inner_mode == NULL) 2249 goto error; 2250 2251 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) 2252 goto error; 2253 2254 x->inner_mode = *inner_mode; 2255 2256 if (x->props.family == AF_INET) 2257 iafamily = AF_INET6; 2258 2259 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily); 2260 if (inner_mode_iaf) { 2261 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL) 2262 x->inner_mode_iaf = *inner_mode_iaf; 2263 } 2264 } 2265 2266 x->type = xfrm_get_type(x->id.proto, family); 2267 if (x->type == NULL) 2268 goto error; 2269 2270 x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload); 2271 2272 err = x->type->init_state(x); 2273 if (err) 2274 goto error; 2275 2276 outer_mode = xfrm_get_mode(x->props.mode, family); 2277 if (!outer_mode) { 2278 err = -EPROTONOSUPPORT; 2279 goto error; 2280 } 2281 2282 x->outer_mode = *outer_mode; 2283 if (init_replay) { 2284 err = xfrm_init_replay(x); 2285 if (err) 2286 goto error; 2287 } 2288 2289 error: 2290 return err; 2291 } 2292 2293 EXPORT_SYMBOL(__xfrm_init_state); 2294 2295 int xfrm_init_state(struct xfrm_state *x) 2296 { 2297 int err; 2298 2299 err = __xfrm_init_state(x, true, false); 2300 if (!err) 2301 x->km.state = XFRM_STATE_VALID; 2302 2303 return err; 2304 } 2305 2306 EXPORT_SYMBOL(xfrm_init_state); 2307 2308 int __net_init xfrm_state_init(struct net *net) 2309 { 2310 unsigned int sz; 2311 2312 if (net_eq(net, &init_net)) 2313 xfrm_state_cache = KMEM_CACHE(xfrm_state, 2314 SLAB_HWCACHE_ALIGN | SLAB_PANIC); 2315 2316 INIT_LIST_HEAD(&net->xfrm.state_all); 2317 2318 sz = sizeof(struct hlist_head) * 8; 2319 2320 net->xfrm.state_bydst = xfrm_hash_alloc(sz); 2321 if (!net->xfrm.state_bydst) 2322 goto out_bydst; 2323 net->xfrm.state_bysrc = xfrm_hash_alloc(sz); 2324 if (!net->xfrm.state_bysrc) 2325 goto out_bysrc; 2326 net->xfrm.state_byspi = xfrm_hash_alloc(sz); 2327 if (!net->xfrm.state_byspi) 2328 goto out_byspi; 2329 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1); 2330 2331 net->xfrm.state_num = 0; 2332 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize); 2333 spin_lock_init(&net->xfrm.xfrm_state_lock); 2334 return 0; 2335 2336 out_byspi: 2337 xfrm_hash_free(net->xfrm.state_bysrc, sz); 2338 out_bysrc: 2339 xfrm_hash_free(net->xfrm.state_bydst, sz); 2340 out_bydst: 2341 return -ENOMEM; 2342 } 2343 2344 void xfrm_state_fini(struct net *net) 2345 { 2346 unsigned int sz; 2347 2348 flush_work(&net->xfrm.state_hash_work); 2349 flush_work(&xfrm_state_gc_work); 2350 xfrm_state_flush(net, 0, false, true); 2351 2352 WARN_ON(!list_empty(&net->xfrm.state_all)); 2353 2354 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); 2355 WARN_ON(!hlist_empty(net->xfrm.state_byspi)); 2356 xfrm_hash_free(net->xfrm.state_byspi, sz); 2357 WARN_ON(!hlist_empty(net->xfrm.state_bysrc)); 2358 xfrm_hash_free(net->xfrm.state_bysrc, sz); 2359 WARN_ON(!hlist_empty(net->xfrm.state_bydst)); 2360 xfrm_hash_free(net->xfrm.state_bydst, sz); 2361 } 2362 2363 #ifdef CONFIG_AUDITSYSCALL 2364 static void xfrm_audit_helper_sainfo(struct xfrm_state *x, 2365 struct audit_buffer *audit_buf) 2366 { 2367 struct xfrm_sec_ctx *ctx = x->security; 2368 u32 spi = ntohl(x->id.spi); 2369 2370 if (ctx) 2371 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", 2372 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); 2373 2374 switch (x->props.family) { 2375 case AF_INET: 2376 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 2377 &x->props.saddr.a4, &x->id.daddr.a4); 2378 break; 2379 case AF_INET6: 2380 audit_log_format(audit_buf, " src=%pI6 dst=%pI6", 2381 x->props.saddr.a6, x->id.daddr.a6); 2382 break; 2383 } 2384 2385 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 2386 } 2387 2388 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family, 2389 struct audit_buffer *audit_buf) 2390 { 2391 const struct iphdr *iph4; 2392 const struct ipv6hdr *iph6; 2393 2394 switch (family) { 2395 case AF_INET: 2396 iph4 = ip_hdr(skb); 2397 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 2398 &iph4->saddr, &iph4->daddr); 2399 break; 2400 case AF_INET6: 2401 iph6 = ipv6_hdr(skb); 2402 audit_log_format(audit_buf, 2403 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x", 2404 &iph6->saddr, &iph6->daddr, 2405 iph6->flow_lbl[0] & 0x0f, 2406 iph6->flow_lbl[1], 2407 iph6->flow_lbl[2]); 2408 break; 2409 } 2410 } 2411 2412 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid) 2413 { 2414 struct audit_buffer *audit_buf; 2415 2416 audit_buf = xfrm_audit_start("SAD-add"); 2417 if (audit_buf == NULL) 2418 return; 2419 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 2420 xfrm_audit_helper_sainfo(x, audit_buf); 2421 audit_log_format(audit_buf, " res=%u", result); 2422 audit_log_end(audit_buf); 2423 } 2424 EXPORT_SYMBOL_GPL(xfrm_audit_state_add); 2425 2426 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid) 2427 { 2428 struct audit_buffer *audit_buf; 2429 2430 audit_buf = xfrm_audit_start("SAD-delete"); 2431 if (audit_buf == NULL) 2432 return; 2433 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 2434 xfrm_audit_helper_sainfo(x, audit_buf); 2435 audit_log_format(audit_buf, " res=%u", result); 2436 audit_log_end(audit_buf); 2437 } 2438 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete); 2439 2440 void xfrm_audit_state_replay_overflow(struct xfrm_state *x, 2441 struct sk_buff *skb) 2442 { 2443 struct audit_buffer *audit_buf; 2444 u32 spi; 2445 2446 audit_buf = xfrm_audit_start("SA-replay-overflow"); 2447 if (audit_buf == NULL) 2448 return; 2449 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 2450 /* don't record the sequence number because it's inherent in this kind 2451 * of audit message */ 2452 spi = ntohl(x->id.spi); 2453 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 2454 audit_log_end(audit_buf); 2455 } 2456 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow); 2457 2458 void xfrm_audit_state_replay(struct xfrm_state *x, 2459 struct sk_buff *skb, __be32 net_seq) 2460 { 2461 struct audit_buffer *audit_buf; 2462 u32 spi; 2463 2464 audit_buf = xfrm_audit_start("SA-replayed-pkt"); 2465 if (audit_buf == NULL) 2466 return; 2467 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 2468 spi = ntohl(x->id.spi); 2469 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 2470 spi, spi, ntohl(net_seq)); 2471 audit_log_end(audit_buf); 2472 } 2473 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay); 2474 2475 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family) 2476 { 2477 struct audit_buffer *audit_buf; 2478 2479 audit_buf = xfrm_audit_start("SA-notfound"); 2480 if (audit_buf == NULL) 2481 return; 2482 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 2483 audit_log_end(audit_buf); 2484 } 2485 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple); 2486 2487 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family, 2488 __be32 net_spi, __be32 net_seq) 2489 { 2490 struct audit_buffer *audit_buf; 2491 u32 spi; 2492 2493 audit_buf = xfrm_audit_start("SA-notfound"); 2494 if (audit_buf == NULL) 2495 return; 2496 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 2497 spi = ntohl(net_spi); 2498 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 2499 spi, spi, ntohl(net_seq)); 2500 audit_log_end(audit_buf); 2501 } 2502 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound); 2503 2504 void xfrm_audit_state_icvfail(struct xfrm_state *x, 2505 struct sk_buff *skb, u8 proto) 2506 { 2507 struct audit_buffer *audit_buf; 2508 __be32 net_spi; 2509 __be32 net_seq; 2510 2511 audit_buf = xfrm_audit_start("SA-icv-failure"); 2512 if (audit_buf == NULL) 2513 return; 2514 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 2515 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) { 2516 u32 spi = ntohl(net_spi); 2517 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 2518 spi, spi, ntohl(net_seq)); 2519 } 2520 audit_log_end(audit_buf); 2521 } 2522 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail); 2523 #endif /* CONFIG_AUDITSYSCALL */ 2524