1*eda14cbcSMatt Macy /* 2*eda14cbcSMatt Macy * CDDL HEADER START 3*eda14cbcSMatt Macy * 4*eda14cbcSMatt Macy * The contents of this file are subject to the terms of the 5*eda14cbcSMatt Macy * Common Development and Distribution License (the "License"). 6*eda14cbcSMatt Macy * You may not use this file except in compliance with the License. 7*eda14cbcSMatt Macy * 8*eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*eda14cbcSMatt Macy * or http://www.opensolaris.org/os/licensing. 10*eda14cbcSMatt Macy * See the License for the specific language governing permissions 11*eda14cbcSMatt Macy * and limitations under the License. 12*eda14cbcSMatt Macy * 13*eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each 14*eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the 16*eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying 17*eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner] 18*eda14cbcSMatt Macy * 19*eda14cbcSMatt Macy * CDDL HEADER END 20*eda14cbcSMatt Macy */ 21*eda14cbcSMatt Macy /* 22*eda14cbcSMatt Macy * Copyright (c) 2014 by Chunwei Chen. All rights reserved. 23*eda14cbcSMatt Macy * Copyright (c) 2016, 2019 by Delphix. All rights reserved. 24*eda14cbcSMatt Macy */ 25*eda14cbcSMatt Macy 26*eda14cbcSMatt Macy #ifndef _ABD_IMPL_H 27*eda14cbcSMatt Macy #define _ABD_IMPL_H 28*eda14cbcSMatt Macy 29*eda14cbcSMatt Macy #include <sys/abd.h> 30*eda14cbcSMatt Macy 31*eda14cbcSMatt Macy #ifdef __cplusplus 32*eda14cbcSMatt Macy extern "C" { 33*eda14cbcSMatt Macy #endif 34*eda14cbcSMatt Macy 35*eda14cbcSMatt Macy typedef enum abd_flags { 36*eda14cbcSMatt Macy ABD_FLAG_LINEAR = 1 << 0, /* is buffer linear (or scattered)? */ 37*eda14cbcSMatt Macy ABD_FLAG_OWNER = 1 << 1, /* does it own its data buffers? */ 38*eda14cbcSMatt Macy ABD_FLAG_META = 1 << 2, /* does this represent FS metadata? */ 39*eda14cbcSMatt Macy ABD_FLAG_MULTI_ZONE = 1 << 3, /* pages split over memory zones */ 40*eda14cbcSMatt Macy ABD_FLAG_MULTI_CHUNK = 1 << 4, /* pages split over multiple chunks */ 41*eda14cbcSMatt Macy ABD_FLAG_LINEAR_PAGE = 1 << 5, /* linear but allocd from page */ 42*eda14cbcSMatt Macy ABD_FLAG_GANG = 1 << 6, /* mult ABDs chained together */ 43*eda14cbcSMatt Macy ABD_FLAG_GANG_FREE = 1 << 7, /* gang ABD is responsible for mem */ 44*eda14cbcSMatt Macy ABD_FLAG_ZEROS = 1 << 8, /* ABD for zero-filled buffer */ 45*eda14cbcSMatt Macy } abd_flags_t; 46*eda14cbcSMatt Macy 47*eda14cbcSMatt Macy typedef enum abd_stats_op { 48*eda14cbcSMatt Macy ABDSTAT_INCR, /* Increase abdstat values */ 49*eda14cbcSMatt Macy ABDSTAT_DECR /* Decrease abdstat values */ 50*eda14cbcSMatt Macy } abd_stats_op_t; 51*eda14cbcSMatt Macy 52*eda14cbcSMatt Macy struct abd { 53*eda14cbcSMatt Macy abd_flags_t abd_flags; 54*eda14cbcSMatt Macy uint_t abd_size; /* excludes scattered abd_offset */ 55*eda14cbcSMatt Macy list_node_t abd_gang_link; 56*eda14cbcSMatt Macy struct abd *abd_parent; 57*eda14cbcSMatt Macy zfs_refcount_t abd_children; 58*eda14cbcSMatt Macy kmutex_t abd_mtx; 59*eda14cbcSMatt Macy union { 60*eda14cbcSMatt Macy struct abd_scatter { 61*eda14cbcSMatt Macy uint_t abd_offset; 62*eda14cbcSMatt Macy #if defined(__FreeBSD__) && defined(_KERNEL) 63*eda14cbcSMatt Macy uint_t abd_chunk_size; 64*eda14cbcSMatt Macy void *abd_chunks[]; 65*eda14cbcSMatt Macy #else 66*eda14cbcSMatt Macy uint_t abd_nents; 67*eda14cbcSMatt Macy struct scatterlist *abd_sgl; 68*eda14cbcSMatt Macy #endif 69*eda14cbcSMatt Macy } abd_scatter; 70*eda14cbcSMatt Macy struct abd_linear { 71*eda14cbcSMatt Macy void *abd_buf; 72*eda14cbcSMatt Macy struct scatterlist *abd_sgl; /* for LINEAR_PAGE */ 73*eda14cbcSMatt Macy } abd_linear; 74*eda14cbcSMatt Macy struct abd_gang { 75*eda14cbcSMatt Macy list_t abd_gang_chain; 76*eda14cbcSMatt Macy } abd_gang; 77*eda14cbcSMatt Macy } abd_u; 78*eda14cbcSMatt Macy }; 79*eda14cbcSMatt Macy 80*eda14cbcSMatt Macy struct scatterlist; /* forward declaration */ 81*eda14cbcSMatt Macy 82*eda14cbcSMatt Macy struct abd_iter { 83*eda14cbcSMatt Macy /* public interface */ 84*eda14cbcSMatt Macy void *iter_mapaddr; /* addr corresponding to iter_pos */ 85*eda14cbcSMatt Macy size_t iter_mapsize; /* length of data valid at mapaddr */ 86*eda14cbcSMatt Macy 87*eda14cbcSMatt Macy /* private */ 88*eda14cbcSMatt Macy abd_t *iter_abd; /* ABD being iterated through */ 89*eda14cbcSMatt Macy size_t iter_pos; 90*eda14cbcSMatt Macy size_t iter_offset; /* offset in current sg/abd_buf, */ 91*eda14cbcSMatt Macy /* abd_offset included */ 92*eda14cbcSMatt Macy struct scatterlist *iter_sg; /* current sg */ 93*eda14cbcSMatt Macy }; 94*eda14cbcSMatt Macy 95*eda14cbcSMatt Macy extern abd_t *abd_zero_scatter; 96*eda14cbcSMatt Macy 97*eda14cbcSMatt Macy abd_t *abd_gang_get_offset(abd_t *, size_t *); 98*eda14cbcSMatt Macy 99*eda14cbcSMatt Macy /* 100*eda14cbcSMatt Macy * OS specific functions 101*eda14cbcSMatt Macy */ 102*eda14cbcSMatt Macy 103*eda14cbcSMatt Macy abd_t *abd_alloc_struct(size_t); 104*eda14cbcSMatt Macy abd_t *abd_get_offset_scatter(abd_t *, size_t); 105*eda14cbcSMatt Macy void abd_free_struct(abd_t *); 106*eda14cbcSMatt Macy void abd_alloc_chunks(abd_t *, size_t); 107*eda14cbcSMatt Macy void abd_free_chunks(abd_t *); 108*eda14cbcSMatt Macy boolean_t abd_size_alloc_linear(size_t); 109*eda14cbcSMatt Macy void abd_update_scatter_stats(abd_t *, abd_stats_op_t); 110*eda14cbcSMatt Macy void abd_update_linear_stats(abd_t *, abd_stats_op_t); 111*eda14cbcSMatt Macy void abd_verify_scatter(abd_t *); 112*eda14cbcSMatt Macy void abd_free_linear_page(abd_t *); 113*eda14cbcSMatt Macy /* OS specific abd_iter functions */ 114*eda14cbcSMatt Macy void abd_iter_init(struct abd_iter *, abd_t *); 115*eda14cbcSMatt Macy boolean_t abd_iter_at_end(struct abd_iter *); 116*eda14cbcSMatt Macy void abd_iter_advance(struct abd_iter *, size_t); 117*eda14cbcSMatt Macy void abd_iter_map(struct abd_iter *); 118*eda14cbcSMatt Macy void abd_iter_unmap(struct abd_iter *); 119*eda14cbcSMatt Macy 120*eda14cbcSMatt Macy /* 121*eda14cbcSMatt Macy * Helper macros 122*eda14cbcSMatt Macy */ 123*eda14cbcSMatt Macy #define ABDSTAT(stat) (abd_stats.stat.value.ui64) 124*eda14cbcSMatt Macy #define ABDSTAT_INCR(stat, val) \ 125*eda14cbcSMatt Macy atomic_add_64(&abd_stats.stat.value.ui64, (val)) 126*eda14cbcSMatt Macy #define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) 127*eda14cbcSMatt Macy #define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) 128*eda14cbcSMatt Macy 129*eda14cbcSMatt Macy #define ABD_SCATTER(abd) (abd->abd_u.abd_scatter) 130*eda14cbcSMatt Macy #define ABD_LINEAR_BUF(abd) (abd->abd_u.abd_linear.abd_buf) 131*eda14cbcSMatt Macy #define ABD_GANG(abd) (abd->abd_u.abd_gang) 132*eda14cbcSMatt Macy 133*eda14cbcSMatt Macy #if defined(_KERNEL) 134*eda14cbcSMatt Macy #if defined(__FreeBSD__) 135*eda14cbcSMatt Macy #define abd_enter_critical(flags) critical_enter() 136*eda14cbcSMatt Macy #define abd_exit_critical(flags) critical_exit() 137*eda14cbcSMatt Macy #else 138*eda14cbcSMatt Macy #define abd_enter_critical(flags) local_irq_save(flags) 139*eda14cbcSMatt Macy #define abd_exit_critical(flags) local_irq_restore(flags) 140*eda14cbcSMatt Macy #endif 141*eda14cbcSMatt Macy #else /* !_KERNEL */ 142*eda14cbcSMatt Macy #define abd_enter_critical(flags) ((void)0) 143*eda14cbcSMatt Macy #define abd_exit_critical(flags) ((void)0) 144*eda14cbcSMatt Macy #endif 145*eda14cbcSMatt Macy 146*eda14cbcSMatt Macy #ifdef __cplusplus 147*eda14cbcSMatt Macy } 148*eda14cbcSMatt Macy #endif 149*eda14cbcSMatt Macy 150*eda14cbcSMatt Macy #endif /* _ABD_IMPL_H */ 151