1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved. 24 */ 25 26 #ifndef _ABD_IMPL_H 27 #define _ABD_IMPL_H 28 29 #include <sys/abd.h> 30 31 #ifdef __cplusplus 32 extern "C" { 33 #endif 34 35 typedef enum abd_flags { 36 ABD_FLAG_LINEAR = 1 << 0, /* is buffer linear (or scattered)? */ 37 ABD_FLAG_OWNER = 1 << 1, /* does it own its data buffers? */ 38 ABD_FLAG_META = 1 << 2, /* does this represent FS metadata? */ 39 ABD_FLAG_MULTI_ZONE = 1 << 3, /* pages split over memory zones */ 40 ABD_FLAG_MULTI_CHUNK = 1 << 4, /* pages split over multiple chunks */ 41 ABD_FLAG_LINEAR_PAGE = 1 << 5, /* linear but allocd from page */ 42 ABD_FLAG_GANG = 1 << 6, /* mult ABDs chained together */ 43 ABD_FLAG_GANG_FREE = 1 << 7, /* gang ABD is responsible for mem */ 44 ABD_FLAG_ZEROS = 1 << 8, /* ABD for zero-filled buffer */ 45 } abd_flags_t; 46 47 typedef enum abd_stats_op { 48 ABDSTAT_INCR, /* Increase abdstat values */ 49 ABDSTAT_DECR /* Decrease abdstat values */ 50 } abd_stats_op_t; 51 52 struct abd { 53 abd_flags_t abd_flags; 54 uint_t abd_size; /* excludes scattered abd_offset */ 55 list_node_t abd_gang_link; 56 struct abd *abd_parent; 57 zfs_refcount_t abd_children; 58 kmutex_t abd_mtx; 59 union { 60 struct abd_scatter { 61 uint_t abd_offset; 62 #if defined(__FreeBSD__) && defined(_KERNEL) 63 uint_t abd_chunk_size; 64 void *abd_chunks[]; 65 #else 66 uint_t abd_nents; 67 struct scatterlist *abd_sgl; 68 #endif 69 } abd_scatter; 70 struct abd_linear { 71 void *abd_buf; 72 struct scatterlist *abd_sgl; /* for LINEAR_PAGE */ 73 } abd_linear; 74 struct abd_gang { 75 list_t abd_gang_chain; 76 } abd_gang; 77 } abd_u; 78 }; 79 80 struct scatterlist; /* forward declaration */ 81 82 struct abd_iter { 83 /* public interface */ 84 void *iter_mapaddr; /* addr corresponding to iter_pos */ 85 size_t iter_mapsize; /* length of data valid at mapaddr */ 86 87 /* private */ 88 abd_t *iter_abd; /* ABD being iterated through */ 89 size_t iter_pos; 90 size_t iter_offset; /* offset in current sg/abd_buf, */ 91 /* abd_offset included */ 92 struct scatterlist *iter_sg; /* current sg */ 93 }; 94 95 extern abd_t *abd_zero_scatter; 96 97 abd_t *abd_gang_get_offset(abd_t *, size_t *); 98 99 /* 100 * OS specific functions 101 */ 102 103 abd_t *abd_alloc_struct(size_t); 104 abd_t *abd_get_offset_scatter(abd_t *, size_t); 105 void abd_free_struct(abd_t *); 106 void abd_alloc_chunks(abd_t *, size_t); 107 void abd_free_chunks(abd_t *); 108 boolean_t abd_size_alloc_linear(size_t); 109 void abd_update_scatter_stats(abd_t *, abd_stats_op_t); 110 void abd_update_linear_stats(abd_t *, abd_stats_op_t); 111 void abd_verify_scatter(abd_t *); 112 void abd_free_linear_page(abd_t *); 113 /* OS specific abd_iter functions */ 114 void abd_iter_init(struct abd_iter *, abd_t *); 115 boolean_t abd_iter_at_end(struct abd_iter *); 116 void abd_iter_advance(struct abd_iter *, size_t); 117 void abd_iter_map(struct abd_iter *); 118 void abd_iter_unmap(struct abd_iter *); 119 120 /* 121 * Helper macros 122 */ 123 #define ABDSTAT(stat) (abd_stats.stat.value.ui64) 124 #define ABDSTAT_INCR(stat, val) \ 125 atomic_add_64(&abd_stats.stat.value.ui64, (val)) 126 #define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) 127 #define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) 128 129 #define ABD_SCATTER(abd) (abd->abd_u.abd_scatter) 130 #define ABD_LINEAR_BUF(abd) (abd->abd_u.abd_linear.abd_buf) 131 #define ABD_GANG(abd) (abd->abd_u.abd_gang) 132 133 #if defined(_KERNEL) 134 #if defined(__FreeBSD__) 135 #define abd_enter_critical(flags) critical_enter() 136 #define abd_exit_critical(flags) critical_exit() 137 #else 138 #define abd_enter_critical(flags) local_irq_save(flags) 139 #define abd_exit_critical(flags) local_irq_restore(flags) 140 #endif 141 #else /* !_KERNEL */ 142 #define abd_enter_critical(flags) ((void)0) 143 #define abd_exit_critical(flags) ((void)0) 144 #endif 145 146 #ifdef __cplusplus 147 } 148 #endif 149 150 #endif /* _ABD_IMPL_H */ 151