1 /* 2 * Copyright 2014, Tushar Gohad, Kevin Greenan, All rights reserved 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are met: 6 * 7 * Redistributions of source code must retain the above copyright notice, this 8 * list of conditions and the following disclaimer. 9 * 10 * Redistributions in binary form must reproduce the above copyright notice, this 11 * list of conditions and the following disclaimer in the documentation and/or 12 * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY 13 * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 14 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 17 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 18 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 21 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 * 24 * liberasurecode frontend API header 25 * 26 * vi: set noai tw=79 ts=4 sw=4: 27 */ 28 29 #ifndef _ERASURECODE_H_ 30 #define _ERASURECODE_H_ 31 32 #include "erasurecode_stdinc.h" 33 #include "erasurecode_version.h" 34 35 #define EC_MAX_FRAGMENTS 32 36 37 #ifdef __cplusplus 38 extern "C" { 39 #endif 40 41 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */ 42 43 typedef enum { 44 EC_BACKEND_NULL = 0, 45 EC_BACKEND_JERASURE_RS_VAND = 1, 46 EC_BACKEND_JERASURE_RS_CAUCHY = 2, 47 EC_BACKEND_FLAT_XOR_HD = 3, 48 EC_BACKEND_ISA_L_RS_VAND = 4, 49 EC_BACKEND_SHSS = 5, 50 EC_BACKEND_LIBERASURECODE_RS_VAND = 6, 51 EC_BACKEND_ISA_L_RS_CAUCHY = 7, 52 EC_BACKENDS_MAX, 53 } ec_backend_id_t; 54 55 /* ~=*=~==~=*=~= EC Fragment metadata - supported checksum types ~=*=~==~=*=~ */ 56 57 /* Checksum types supported for fragment metadata stored in each fragment */ 58 typedef enum { 59 CHKSUM_NONE = 1, 60 CHKSUM_CRC32 = 2, 61 CHKSUM_MD5 = 3, 62 CHKSUM_TYPES_MAX, 63 } ec_checksum_type_t; 64 65 /* =~=*=~==~=*=~== EC Arguments - Common and backend-specific =~=*=~==~=*=~== */ 66 67 /** 68 * Common and backend-specific args 69 * to be passed to liberasurecode_instance_create() 70 */ 71 struct ec_args { 72 int k; /* number of data fragments */ 73 int m; /* number of parity fragments */ 74 int w; /* word size, in bits (optional) */ 75 int hd; /* hamming distance (=m for Reed-Solomon) */ 76 77 union { 78 struct { 79 uint64_t arg1; /* sample arg */ 80 } null_args; /* args specific to the null codes */ 81 struct { 82 uint64_t x, y; /* reserved for future expansion */ 83 uint64_t z, a; /* reserved for future expansion */ 84 } reserved; 85 } priv_args1; 86 87 void *priv_args2; /** flexible placeholder for 88 * future backend args */ 89 ec_checksum_type_t ct; /* fragment checksum type */ 90 }; 91 92 /* =~=*=~==~=*=~== liberasurecode frontend API functions =~=*=~==~=~=*=~==~= */ 93 94 /* liberasurecode frontend API functions */ 95 96 /** 97 * Checks if a given backend is available. 98 * 99 * @param backend_id - one of the supported backends. 100 * 101 * @returns 1 if a backend is available; 0 otherwise 102 */ 103 int liberasurecode_backend_available(const ec_backend_id_t backend_id); 104 105 /** 106 * Create a liberasurecode instance and return a descriptor 107 * for use with EC operations (encode, decode, reconstruct) 108 * 109 * @param id - one of the supported backends as 110 * defined by ec_backend_id_t 111 * @param ec_args - arguments to the EC backend 112 * arguments common to all backends 113 * k - number of data fragments 114 * m - number of parity fragments 115 * w - word size, in bits 116 * hd - hamming distance (=m for Reed-Solomon) 117 * ct - fragment checksum type (stored with the fragment metadata) 118 * backend-specific arguments 119 * null_args - arguments for the null backend 120 * flat_xor_hd, jerasure do not require any special args 121 * 122 * @return liberasurecode instance descriptor (int > 0) 123 */ 124 int liberasurecode_instance_create(const ec_backend_id_t id, 125 struct ec_args *args); 126 127 /** 128 * Close a liberasurecode instance 129 * 130 * @param desc - liberasurecode descriptor to close 131 * 132 * @return 0 on success, otherwise non-zero error code 133 */ 134 int liberasurecode_instance_destroy(int desc); 135 136 137 /** 138 * Erasure encode a data buffer 139 * 140 * @param desc - liberasurecode descriptor/handle 141 * from liberasurecode_instance_create() 142 * @param orig_data - data to encode 143 * @param orig_data_size - length of data to encode 144 * @param encoded_data - pointer to _output_ array (char **) of k data 145 * fragments (char *), allocated by the callee 146 * @param encoded_parity - pointer to _output_ array (char **) of m parity 147 * fragments (char *), allocated by the callee 148 * @param fragment_len - pointer to _output_ length of each fragment, assuming 149 * all fragments are the same length 150 * 151 * @return 0 on success, -error code otherwise 152 */ 153 int liberasurecode_encode(int desc, 154 const char *orig_data, uint64_t orig_data_size, /* input */ 155 char ***encoded_data, char ***encoded_parity, /* output */ 156 uint64_t *fragment_len); /* output */ 157 158 /** 159 * Cleanup structures allocated by librasurecode_encode 160 * 161 * The caller has no context, so cannot safely free memory 162 * allocated by liberasurecode, so it must pass the 163 * deallocation responsibility back to liberasurecode. 164 * 165 * @param desc - liberasurecode descriptor/handle 166 * from liberasurecode_instance_create() 167 * @param encoded_data - (char **) array of k data 168 * fragments (char *), allocated by liberasurecode_encode 169 * @param encoded_parity - (char **) array of m parity 170 * fragments (char *), allocated by liberasurecode_encode 171 * 172 * @return 0 in success; -error otherwise 173 */ 174 int liberasurecode_encode_cleanup(int desc, char **encoded_data, 175 char **encoded_parity); 176 177 /** 178 * Reconstruct original data from a set of k encoded fragments 179 * 180 * @param desc - liberasurecode descriptor/handle 181 * from liberasurecode_instance_create() 182 * @param fragments - erasure encoded fragments (> = k) 183 * @param num_fragments - number of fragments being passed in 184 * @param fragment_len - length of each fragment (assume they are the same) 185 * @param force_metadata_checks - force fragment metadata checks (default: 0) 186 * @param out_data - _output_ pointer to decoded data 187 * @param out_data_len - _output_ length of decoded output 188 * (both output data pointers are allocated by liberasurecode, 189 * caller invokes liberasurecode_decode_clean() after it has 190 * read decoded data in 'out_data') 191 * 192 * @return 0 on success, -error code otherwise 193 */ 194 int liberasurecode_decode(int desc, 195 char **available_fragments, /* input */ 196 int num_fragments, uint64_t fragment_len, /* input */ 197 int force_metadata_checks, /* input */ 198 char **out_data, uint64_t *out_data_len); /* output */ 199 200 /** 201 * Cleanup structures allocated by librasurecode_decode 202 * 203 * The caller has no context, so cannot safely free memory 204 * allocated by liberasurecode, so it must pass the 205 * deallocation responsibility back to liberasurecode. 206 * 207 * @param desc - liberasurecode descriptor/handle 208 * from liberasurecode_instance_create() 209 * @param data - (char *) buffer of data decoded by librasurecode_decode 210 * 211 * @return 0 on success; -error otherwise 212 */ 213 int liberasurecode_decode_cleanup(int desc, char *data); 214 215 /** 216 * Reconstruct a missing fragment from a subset of available fragments 217 * 218 * @param desc - liberasurecode descriptor/handle 219 * from liberasurecode_instance_create() 220 * @param available_fragments - erasure encoded fragments 221 * @param num_fragments - number of fragments being passed in 222 * @param fragment_len - size in bytes of the fragments 223 * @param destination_idx - missing idx to reconstruct 224 * @param out_fragment - output of reconstruct 225 * 226 * @return 0 on success, -error code otherwise 227 */ 228 int liberasurecode_reconstruct_fragment(int desc, 229 char **available_fragments, /* input */ 230 int num_fragments, uint64_t fragment_len, /* input */ 231 int destination_idx, /* input */ 232 char* out_fragment); /* output */ 233 234 /** 235 * Return a list of lists with valid rebuild indexes given 236 * a list of missing indexes. 237 * 238 * @desc: liberasurecode instance descriptor (obtained with 239 * liberasurecode_instance_create) 240 * @fragments_to_reconstruct list of indexes to reconstruct 241 * @fragments_to_exclude list of indexes to exclude from 242 * reconstruction equation 243 * @fragments_needed list of fragments needed to reconstruct 244 * fragments in fragments_to_reconstruct 245 * 246 * @return 0 on success, non-zero on error 247 */ 248 int liberasurecode_fragments_needed(int desc, 249 int *fragments_to_reconstruct, 250 int *fragments_to_exclude, 251 int *fragments_needed); 252 253 254 /* ==~=*=~==~=*=~== liberasurecode fragment metadata routines ==~*==~=*=~==~ */ 255 256 #define LIBERASURECODE_MAX_CHECKSUM_LEN 8 257 typedef struct __attribute__((__packed__)) 258 fragment_metadata 259 { 260 uint32_t idx; /* 4 */ 261 uint32_t size; /* 4 */ 262 uint32_t frag_backend_metadata_size; /* 4 */ 263 uint64_t orig_data_size; /* 8 */ 264 uint8_t chksum_type; /* 1 */ 265 uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */ 266 uint8_t chksum_mismatch; /* 1 */ 267 uint8_t backend_id; /* 1 */ 268 uint32_t backend_version; /* 4 */ 269 } fragment_metadata_t; 270 271 /** 272 * Get opaque metadata for a fragment. The metadata is opaque to the 273 * client, but meaningful to the underlying library. It is used to verify 274 * stripes in verify_stripe_metadata(). 275 * 276 * @param fragment - fragment data pointer 277 * @param fragment_metadata - pointer to allocated buffer of size at least 278 * sizeof(struct fragment_metadata) to hold fragment metadata struct 279 * 280 * @return 0 on success, non-zero on error 281 */ 282 //EDL: This needs to be implemented 283 int liberasurecode_get_fragment_metadata(char *fragment, 284 fragment_metadata_t *fragment_metadata); 285 286 /** 287 * Verify that the specified pointer points to a well formed fragment that can 288 * be processed by both this instance of liberasurecode and the specified 289 * backend. 290 * 291 * @param desc - liberasurecode descriptor/handle 292 * from liberasurecode_instance_create() 293 * @param fragment - fragment to verify 294 * 295 * @return 1 if fragment validation fails, 0 otherwise. 296 */ 297 int is_invalid_fragment(int desc, char *fragment); 298 299 /** 300 * Verify a subset of fragments generated by encode() 301 * 302 * @param desc - liberasurecode descriptor/handle 303 * from liberasurecode_instance_create() 304 * @param fragments - fragments part of the EC stripe to verify 305 * @param num_fragments - number of fragments part of the EC stripe 306 * 307 * @return 1 if stripe checksum verification is successful, 0 otherwise 308 */ 309 int liberasurecode_verify_stripe_metadata(int desc, 310 char **fragments, int num_fragments); 311 312 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ 313 314 /** 315 * liberasurecode fragment header definition 316 * 317 * Prevent the compiler from padding this by using the __packed__ keyword 318 */ 319 320 #define LIBERASURECODE_FRAG_HEADER_MAGIC 0xb0c5ecc 321 #define LIBERASURECODE_MAX_CHECKSUM_LEN 8 /* quad words */ 322 323 typedef struct __attribute__((__packed__)) fragment_header_s 324 { 325 fragment_metadata_t meta; /* 59 bytes */ 326 uint32_t magic; /* 4 bytes */ 327 uint32_t libec_version; /* 4 bytes */ 328 uint32_t metadata_chksum; /* 4 bytes */ 329 // We must be aligned to 16-byte boundaries 330 // So, size this array accordingly 331 uint8_t aligned_padding[9]; 332 } fragment_header_t; 333 334 #define FRAGSIZE_2_BLOCKSIZE(fragment_size) \ 335 (fragment_size - sizeof(fragment_header_t)) 336 337 /* ==~=*=~===~=*=~==~=*=~== liberasurecode Helpers ==~*==~=*=~==~=~=*=~==~= */ 338 339 /** 340 * This computes the aligned size of a buffer passed into 341 * the encode function. The encode function must pad fragments 342 * to be algined with the word size (w) and the last fragment also 343 * needs to be aligned. This computes the sum of the algined fragment 344 * sizes for a given buffer to encode. 345 * 346 * @param desc - liberasurecode descriptor/handle 347 * from liberasurecode_instance_create() 348 * @param data_len - original data length in bytes 349 * 350 * @return aligned length, or -error code on error 351 */ 352 int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len); 353 354 /** 355 * This will return the minimum encode size, which is the minimum 356 * buffer size that can be encoded. 357 * 358 * @param desc - liberasurecode descriptor/handle 359 * from liberasurecode_instance_create() 360 * 361 * @return minimum data length length, or -error code on error 362 */ 363 int liberasurecode_get_minimum_encode_size(int desc); 364 365 /** 366 * This will return the fragment size, which is each fragment data 367 * length the backend will allocate when encoding. 368 * 369 * @param desc - liberasurecode descriptor/handle 370 * from liberasurecode_instance_create() 371 * @param data_len - original data length in bytes 372 * 373 * @return fragment size - sizeof(fragment_header) + size 374 * + frag_backend_metadata_size 375 * if an error, return value will be negative 376 */ 377 int liberasurecode_get_fragment_size(int desc, int data_len); 378 379 /** 380 * This will return the liberasurecode version for the descriptor 381 * 382 * @return version uint32_t - from erasurecode_version.h 383 */ 384 385 uint32_t liberasurecode_get_version(); 386 387 /* ==~=*=~===~=*=~==~=*=~== liberasurecode Error codes =~=*=~==~=~=*=~==~== */ 388 389 /* Error codes */ 390 typedef enum { 391 EBACKENDNOTSUPP = 200, 392 EECMETHODNOTIMPL = 201, 393 EBACKENDINITERR = 202, 394 EBACKENDINUSE = 203, 395 EBACKENDNOTAVAIL = 204, 396 EBADCHKSUM = 205, 397 EINVALIDPARAMS = 206, 398 EBADHEADER = 207, 399 EINSUFFFRAGS = 208, 400 } LIBERASURECODE_ERROR_CODES; 401 402 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */ 403 404 #ifdef __cplusplus 405 } 406 #endif 407 408 #endif // _ERASURECODE_H_ 409