1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 14 * Copyright (c) 2016 by Delphix. All rights reserved. 15 */ 16 17 /* 18 * See abd.c for a general overview of the arc buffered data (ABD). 19 * 20 * Using a large proportion of scattered ABDs decreases ARC fragmentation since 21 * when we are at the limit of allocatable space, using equal-size chunks will 22 * allow us to quickly reclaim enough space for a new large allocation (assuming 23 * it is also scattered). 24 * 25 * ABDs are allocated scattered by default unless the caller uses 26 * abd_alloc_linear() or zfs_abd_scatter_enabled is disabled. 27 */ 28 29 #include <sys/abd_impl.h> 30 #include <sys/param.h> 31 #include <sys/types.h> 32 #include <sys/zio.h> 33 #include <sys/zfs_context.h> 34 #include <sys/zfs_znode.h> 35 36 typedef struct abd_stats { 37 kstat_named_t abdstat_struct_size; 38 kstat_named_t abdstat_scatter_cnt; 39 kstat_named_t abdstat_scatter_data_size; 40 kstat_named_t abdstat_scatter_chunk_waste; 41 kstat_named_t abdstat_linear_cnt; 42 kstat_named_t abdstat_linear_data_size; 43 } abd_stats_t; 44 45 static abd_stats_t abd_stats = { 46 /* Amount of memory occupied by all of the abd_t struct allocations */ 47 { "struct_size", KSTAT_DATA_UINT64 }, 48 /* 49 * The number of scatter ABDs which are currently allocated, excluding 50 * ABDs which don't own their data (for instance the ones which were 51 * allocated through abd_get_offset()). 52 */ 53 { "scatter_cnt", KSTAT_DATA_UINT64 }, 54 /* Amount of data stored in all scatter ABDs tracked by scatter_cnt */ 55 { "scatter_data_size", KSTAT_DATA_UINT64 }, 56 /* 57 * The amount of space wasted at the end of the last chunk across all 58 * scatter ABDs tracked by scatter_cnt. 59 */ 60 { "scatter_chunk_waste", KSTAT_DATA_UINT64 }, 61 /* 62 * The number of linear ABDs which are currently allocated, excluding 63 * ABDs which don't own their data (for instance the ones which were 64 * allocated through abd_get_offset() and abd_get_from_buf()). If an 65 * ABD takes ownership of its buf then it will become tracked. 66 */ 67 { "linear_cnt", KSTAT_DATA_UINT64 }, 68 /* Amount of data stored in all linear ABDs tracked by linear_cnt */ 69 { "linear_data_size", KSTAT_DATA_UINT64 }, 70 }; 71 72 struct { 73 wmsum_t abdstat_struct_size; 74 wmsum_t abdstat_scatter_cnt; 75 wmsum_t abdstat_scatter_data_size; 76 wmsum_t abdstat_scatter_chunk_waste; 77 wmsum_t abdstat_linear_cnt; 78 wmsum_t abdstat_linear_data_size; 79 } abd_sums; 80 81 /* 82 * zfs_abd_scatter_min_size is the minimum allocation size to use scatter 83 * ABD's for. Smaller allocations will use linear ABD's which use 84 * zio_[data_]buf_alloc(). 85 * 86 * Scatter ABD's use at least one page each, so sub-page allocations waste 87 * some space when allocated as scatter (e.g. 2KB scatter allocation wastes 88 * half of each page). Using linear ABD's for small allocations means that 89 * they will be put on slabs which contain many allocations. 90 * 91 * Linear ABDs for multi-page allocations are easier to use, and in some cases 92 * it allows to avoid buffer copying. But allocation and especially free 93 * of multi-page linear ABDs are expensive operations due to KVA mapping and 94 * unmapping, and with time they cause KVA fragmentations. 95 */ 96 static size_t zfs_abd_scatter_min_size = PAGE_SIZE + 1; 97 98 #if defined(_KERNEL) 99 SYSCTL_DECL(_vfs_zfs); 100 101 SYSCTL_INT(_vfs_zfs, OID_AUTO, abd_scatter_enabled, CTLFLAG_RWTUN, 102 &zfs_abd_scatter_enabled, 0, "Enable scattered ARC data buffers"); 103 SYSCTL_ULONG(_vfs_zfs, OID_AUTO, abd_scatter_min_size, CTLFLAG_RWTUN, 104 &zfs_abd_scatter_min_size, 0, "Minimum size of scatter allocations."); 105 #endif 106 107 kmem_cache_t *abd_chunk_cache; 108 static kstat_t *abd_ksp; 109 110 /* 111 * We use a scattered SPA_MAXBLOCKSIZE sized ABD whose chunks are 112 * just a single zero'd page-sized buffer. This allows us to conserve 113 * memory by only using a single zero buffer for the scatter chunks. 114 */ 115 abd_t *abd_zero_scatter = NULL; 116 static char *abd_zero_buf = NULL; 117 118 static uint_t 119 abd_chunkcnt_for_bytes(size_t size) 120 { 121 return ((size + PAGE_MASK) >> PAGE_SHIFT); 122 } 123 124 static inline uint_t 125 abd_scatter_chunkcnt(abd_t *abd) 126 { 127 ASSERT(!abd_is_linear(abd)); 128 return (abd_chunkcnt_for_bytes( 129 ABD_SCATTER(abd).abd_offset + abd->abd_size)); 130 } 131 132 boolean_t 133 abd_size_alloc_linear(size_t size) 134 { 135 return (!zfs_abd_scatter_enabled || size < zfs_abd_scatter_min_size); 136 } 137 138 void 139 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op) 140 { 141 uint_t n = abd_scatter_chunkcnt(abd); 142 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 143 int waste = (n << PAGE_SHIFT) - abd->abd_size; 144 if (op == ABDSTAT_INCR) { 145 ABDSTAT_BUMP(abdstat_scatter_cnt); 146 ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size); 147 ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste); 148 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE); 149 } else { 150 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt); 151 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size); 152 ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste); 153 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE); 154 } 155 } 156 157 void 158 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op) 159 { 160 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR); 161 if (op == ABDSTAT_INCR) { 162 ABDSTAT_BUMP(abdstat_linear_cnt); 163 ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size); 164 } else { 165 ABDSTAT_BUMPDOWN(abdstat_linear_cnt); 166 ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size); 167 } 168 } 169 170 void 171 abd_verify_scatter(abd_t *abd) 172 { 173 uint_t i, n; 174 175 /* 176 * There is no scatter linear pages in FreeBSD so there is 177 * an error if the ABD has been marked as a linear page. 178 */ 179 ASSERT(!abd_is_linear_page(abd)); 180 ASSERT3U(ABD_SCATTER(abd).abd_offset, <, PAGE_SIZE); 181 n = abd_scatter_chunkcnt(abd); 182 for (i = 0; i < n; i++) { 183 ASSERT3P(ABD_SCATTER(abd).abd_chunks[i], !=, NULL); 184 } 185 } 186 187 void 188 abd_alloc_chunks(abd_t *abd, size_t size) 189 { 190 uint_t i, n; 191 192 n = abd_chunkcnt_for_bytes(size); 193 for (i = 0; i < n; i++) { 194 ABD_SCATTER(abd).abd_chunks[i] = 195 kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE); 196 } 197 } 198 199 void 200 abd_free_chunks(abd_t *abd) 201 { 202 uint_t i, n; 203 204 n = abd_scatter_chunkcnt(abd); 205 for (i = 0; i < n; i++) { 206 kmem_cache_free(abd_chunk_cache, 207 ABD_SCATTER(abd).abd_chunks[i]); 208 } 209 } 210 211 abd_t * 212 abd_alloc_struct_impl(size_t size) 213 { 214 uint_t chunkcnt = abd_chunkcnt_for_bytes(size); 215 /* 216 * In the event we are allocating a gang ABD, the size passed in 217 * will be 0. We must make sure to set abd_size to the size of an 218 * ABD struct as opposed to an ABD scatter with 0 chunks. The gang 219 * ABD struct allocation accounts for an additional 24 bytes over 220 * a scatter ABD with 0 chunks. 221 */ 222 size_t abd_size = MAX(sizeof (abd_t), 223 offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt])); 224 abd_t *abd = kmem_alloc(abd_size, KM_PUSHPAGE); 225 ASSERT3P(abd, !=, NULL); 226 ABDSTAT_INCR(abdstat_struct_size, abd_size); 227 228 return (abd); 229 } 230 231 void 232 abd_free_struct_impl(abd_t *abd) 233 { 234 uint_t chunkcnt = abd_is_linear(abd) || abd_is_gang(abd) ? 0 : 235 abd_scatter_chunkcnt(abd); 236 ssize_t size = MAX(sizeof (abd_t), 237 offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt])); 238 kmem_free(abd, size); 239 ABDSTAT_INCR(abdstat_struct_size, -size); 240 } 241 242 /* 243 * Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where 244 * each chunk in the scatterlist will be set to abd_zero_buf. 245 */ 246 static void 247 abd_alloc_zero_scatter(void) 248 { 249 uint_t i, n; 250 251 n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE); 252 abd_zero_buf = kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE); 253 bzero(abd_zero_buf, PAGE_SIZE); 254 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE); 255 256 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER | ABD_FLAG_ZEROS; 257 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE; 258 259 ABD_SCATTER(abd_zero_scatter).abd_offset = 0; 260 261 for (i = 0; i < n; i++) { 262 ABD_SCATTER(abd_zero_scatter).abd_chunks[i] = 263 abd_zero_buf; 264 } 265 266 ABDSTAT_BUMP(abdstat_scatter_cnt); 267 ABDSTAT_INCR(abdstat_scatter_data_size, PAGE_SIZE); 268 } 269 270 static void 271 abd_free_zero_scatter(void) 272 { 273 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt); 274 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)PAGE_SIZE); 275 276 abd_free_struct(abd_zero_scatter); 277 abd_zero_scatter = NULL; 278 kmem_cache_free(abd_chunk_cache, abd_zero_buf); 279 } 280 281 static int 282 abd_kstats_update(kstat_t *ksp, int rw) 283 { 284 abd_stats_t *as = ksp->ks_data; 285 286 if (rw == KSTAT_WRITE) 287 return (EACCES); 288 as->abdstat_struct_size.value.ui64 = 289 wmsum_value(&abd_sums.abdstat_struct_size); 290 as->abdstat_scatter_cnt.value.ui64 = 291 wmsum_value(&abd_sums.abdstat_scatter_cnt); 292 as->abdstat_scatter_data_size.value.ui64 = 293 wmsum_value(&abd_sums.abdstat_scatter_data_size); 294 as->abdstat_scatter_chunk_waste.value.ui64 = 295 wmsum_value(&abd_sums.abdstat_scatter_chunk_waste); 296 as->abdstat_linear_cnt.value.ui64 = 297 wmsum_value(&abd_sums.abdstat_linear_cnt); 298 as->abdstat_linear_data_size.value.ui64 = 299 wmsum_value(&abd_sums.abdstat_linear_data_size); 300 return (0); 301 } 302 303 void 304 abd_init(void) 305 { 306 abd_chunk_cache = kmem_cache_create("abd_chunk", PAGE_SIZE, 0, 307 NULL, NULL, NULL, NULL, 0, KMC_NODEBUG); 308 309 wmsum_init(&abd_sums.abdstat_struct_size, 0); 310 wmsum_init(&abd_sums.abdstat_scatter_cnt, 0); 311 wmsum_init(&abd_sums.abdstat_scatter_data_size, 0); 312 wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0); 313 wmsum_init(&abd_sums.abdstat_linear_cnt, 0); 314 wmsum_init(&abd_sums.abdstat_linear_data_size, 0); 315 316 abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED, 317 sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 318 if (abd_ksp != NULL) { 319 abd_ksp->ks_data = &abd_stats; 320 abd_ksp->ks_update = abd_kstats_update; 321 kstat_install(abd_ksp); 322 } 323 324 abd_alloc_zero_scatter(); 325 } 326 327 void 328 abd_fini(void) 329 { 330 abd_free_zero_scatter(); 331 332 if (abd_ksp != NULL) { 333 kstat_delete(abd_ksp); 334 abd_ksp = NULL; 335 } 336 337 wmsum_fini(&abd_sums.abdstat_struct_size); 338 wmsum_fini(&abd_sums.abdstat_scatter_cnt); 339 wmsum_fini(&abd_sums.abdstat_scatter_data_size); 340 wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste); 341 wmsum_fini(&abd_sums.abdstat_linear_cnt); 342 wmsum_fini(&abd_sums.abdstat_linear_data_size); 343 344 kmem_cache_destroy(abd_chunk_cache); 345 abd_chunk_cache = NULL; 346 } 347 348 void 349 abd_free_linear_page(abd_t *abd) 350 { 351 /* 352 * FreeBSD does not have scatter linear pages 353 * so there is an error. 354 */ 355 VERIFY(0); 356 } 357 358 /* 359 * If we're going to use this ABD for doing I/O using the block layer, the 360 * consumer of the ABD data doesn't care if it's scattered or not, and we don't 361 * plan to store this ABD in memory for a long period of time, we should 362 * allocate the ABD type that requires the least data copying to do the I/O. 363 * 364 * Currently this is linear ABDs, however if ldi_strategy() can ever issue I/Os 365 * using a scatter/gather list we should switch to that and replace this call 366 * with vanilla abd_alloc(). 367 */ 368 abd_t * 369 abd_alloc_for_io(size_t size, boolean_t is_metadata) 370 { 371 return (abd_alloc_linear(size, is_metadata)); 372 } 373 374 abd_t * 375 abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off, 376 size_t size) 377 { 378 abd_verify(sabd); 379 ASSERT3U(off, <=, sabd->abd_size); 380 381 size_t new_offset = ABD_SCATTER(sabd).abd_offset + off; 382 size_t chunkcnt = abd_chunkcnt_for_bytes( 383 (new_offset & PAGE_MASK) + size); 384 385 ASSERT3U(chunkcnt, <=, abd_scatter_chunkcnt(sabd)); 386 387 /* 388 * If an abd struct is provided, it is only the minimum size. If we 389 * need additional chunks, we need to allocate a new struct. 390 */ 391 if (abd != NULL && 392 offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]) > 393 sizeof (abd_t)) { 394 abd = NULL; 395 } 396 397 if (abd == NULL) 398 abd = abd_alloc_struct(chunkcnt << PAGE_SHIFT); 399 400 /* 401 * Even if this buf is filesystem metadata, we only track that 402 * if we own the underlying data buffer, which is not true in 403 * this case. Therefore, we don't ever use ABD_FLAG_META here. 404 */ 405 406 ABD_SCATTER(abd).abd_offset = new_offset & PAGE_MASK; 407 408 /* Copy the scatterlist starting at the correct offset */ 409 (void) memcpy(&ABD_SCATTER(abd).abd_chunks, 410 &ABD_SCATTER(sabd).abd_chunks[new_offset >> PAGE_SHIFT], 411 chunkcnt * sizeof (void *)); 412 413 return (abd); 414 } 415 416 /* 417 * Initialize the abd_iter. 418 */ 419 void 420 abd_iter_init(struct abd_iter *aiter, abd_t *abd) 421 { 422 ASSERT(!abd_is_gang(abd)); 423 abd_verify(abd); 424 aiter->iter_abd = abd; 425 aiter->iter_pos = 0; 426 aiter->iter_mapaddr = NULL; 427 aiter->iter_mapsize = 0; 428 } 429 430 /* 431 * This is just a helper function to see if we have exhausted the 432 * abd_iter and reached the end. 433 */ 434 boolean_t 435 abd_iter_at_end(struct abd_iter *aiter) 436 { 437 return (aiter->iter_pos == aiter->iter_abd->abd_size); 438 } 439 440 /* 441 * Advance the iterator by a certain amount. Cannot be called when a chunk is 442 * in use. This can be safely called when the aiter has already exhausted, in 443 * which case this does nothing. 444 */ 445 void 446 abd_iter_advance(struct abd_iter *aiter, size_t amount) 447 { 448 ASSERT3P(aiter->iter_mapaddr, ==, NULL); 449 ASSERT0(aiter->iter_mapsize); 450 451 /* There's nothing left to advance to, so do nothing */ 452 if (abd_iter_at_end(aiter)) 453 return; 454 455 aiter->iter_pos += amount; 456 } 457 458 /* 459 * Map the current chunk into aiter. This can be safely called when the aiter 460 * has already exhausted, in which case this does nothing. 461 */ 462 void 463 abd_iter_map(struct abd_iter *aiter) 464 { 465 void *paddr; 466 467 ASSERT3P(aiter->iter_mapaddr, ==, NULL); 468 ASSERT0(aiter->iter_mapsize); 469 470 /* There's nothing left to iterate over, so do nothing */ 471 if (abd_iter_at_end(aiter)) 472 return; 473 474 abd_t *abd = aiter->iter_abd; 475 size_t offset = aiter->iter_pos; 476 if (abd_is_linear(abd)) { 477 aiter->iter_mapsize = abd->abd_size - offset; 478 paddr = ABD_LINEAR_BUF(abd); 479 } else { 480 offset += ABD_SCATTER(abd).abd_offset; 481 paddr = ABD_SCATTER(abd).abd_chunks[offset >> PAGE_SHIFT]; 482 offset &= PAGE_MASK; 483 aiter->iter_mapsize = MIN(PAGE_SIZE - offset, 484 abd->abd_size - aiter->iter_pos); 485 } 486 aiter->iter_mapaddr = (char *)paddr + offset; 487 } 488 489 /* 490 * Unmap the current chunk from aiter. This can be safely called when the aiter 491 * has already exhausted, in which case this does nothing. 492 */ 493 void 494 abd_iter_unmap(struct abd_iter *aiter) 495 { 496 if (!abd_iter_at_end(aiter)) { 497 ASSERT3P(aiter->iter_mapaddr, !=, NULL); 498 ASSERT3U(aiter->iter_mapsize, >, 0); 499 } 500 501 aiter->iter_mapaddr = NULL; 502 aiter->iter_mapsize = 0; 503 } 504 505 void 506 abd_cache_reap_now(void) 507 { 508 kmem_cache_reap_soon(abd_chunk_cache); 509 } 510