1 #pragma once 2 #ifndef IWFSMFILE_H 3 #define IWFSMFILE_H 4 5 /************************************************************************************************** 6 * IOWOW library 7 * 8 * MIT License 9 * 10 * Copyright (c) 2012-2021 Softmotions Ltd <info@softmotions.com> 11 * 12 * Permission is hereby granted, free of charge, to any person obtaining a copy 13 * of this software and associated documentation files (the "Software"), to deal 14 * in the Software without restriction, including without limitation the rights 15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 * copies of the Software, and to permit persons to whom the Software is 17 * furnished to do so, subject to the following conditions: 18 * 19 * The above copyright notice and this permission notice shall be included in all 20 * copies or substantial portions of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 * SOFTWARE. 29 *************************************************************************************************/ 30 31 /** @file 32 * @brief Auto-expandable file with support of reader/writer address space 33 locking and free space block management using bitmaps. 34 * @author Anton Adamansky (adamansky@softmotions.com) 35 * 36 * @note Before using API of this module you should call 37 * `iw_init(void)` iowow module initialization routine. 38 * 39 * <strong>Features:</strong> 40 * 41 * - Address blocks allocation and deallocation using bitmaps. 42 * - Read/write file address space locking. 43 * - Tunable file expansion policies. 44 * - Read/write methods locking option in multithreaded environment. 45 * - File shrinking/truncation support. 46 * - A number mmaped regions can be registered in the file's address space. 47 * These regions used in read/write operation and automatically maintained 48 * during file resize operations. 49 * 50 * File operations implemented as function pointers contained 51 * in `IWFS_FSM` `C` structure. 52 * The `iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts)` opens file 53 * and initializes a given `IWFS_FSM` structure. 54 * 55 * <strong>File format:</strong> 56 * @verbatim 57 [FSM_CTL_MAGICK u32][block pow u8] 58 [bmoffset u64][bmlength u64] 59 [crzsum u64][crznum u32][crszvar u64][reserved u256] 60 [custom header size u32][custom header data...] 61 [fsm data...] @endverbatim 62 * 63 * <strong>where:</strong> 64 * 65 * - <b>FSM_CTL_MAGICK:</b> Free-space file magic number (32 bit) 66 * - <b>block pow:</b> Block size as power of `2` Eg: `6` means `64` bit block 67 size. (8 bit) 68 * - <b>bmoffset:</b> Free space bitmap area offset in bytes (64 bit) 69 * - <b>bmlength:</b> Free space bitmap area length. (64 bit) 70 * - <b>crzsum:</b> Number of allocated blocks. (64 bit) 71 * - <b>crznum:</b> Number of all allocated continuous areas. (32 bit) 72 * - <b>crszvar</b> Allocated areas length standard variance (deviation^2 * N) (64 bit) 73 * - <b>reserved:</b> Reserved space. 74 * - <b>custom header size:</b> Length of custom header area. See 75 `IWFS_FSM::writehdr` and `IWFS_FSM::readhdr` 76 */ 77 78 #include "iwexfile.h" 79 #include <stdbool.h> 80 #include <math.h> 81 82 IW_EXTERN_C_START 83 84 /** Free space allocation flags 85 * @see IWFS_FSM::allocate 86 */ 87 typedef uint8_t iwfs_fsm_aflags; 88 89 /** Use default allocation settings */ 90 #define IWFSM_ALLOC_DEFAULTS ((iwfs_fsm_aflags) 0x00U) 91 92 /** Do not @em overallocate a requested free space in order to reduce fragmentation */ 93 #define IWFSM_ALLOC_NO_OVERALLOCATE ((iwfs_fsm_aflags) 0x01U) 94 95 /** Do not extend the file and its bitmap free space mapping in the case if 96 * file size expansion is required. 97 * In this case the `IWFS_ERROR_NO_FREE_SPACE` error will be raised.*/ 98 #define IWFSM_ALLOC_NO_EXTEND ((iwfs_fsm_aflags) 0x02U) 99 100 /** Force offset of an allocated space to be page aligned. */ 101 #define IWFSM_ALLOC_PAGE_ALIGNED ((iwfs_fsm_aflags) 0x04U) 102 103 /** Do not collect internal allocation stats for this allocation. */ 104 #define IWFSM_ALLOC_NO_STATS ((iwfs_fsm_aflags) 0x08U) 105 106 /** Force all of the allocated address space backed by real file address space. */ 107 #define IWFSM_SOLID_ALLOCATED_SPACE ((iwfs_fsm_aflags) 0x10U) 108 109 /** Do msync of bitmap allocation index. */ 110 #define IWFSM_SYNC_BMAP ((iwfs_fsm_aflags) 0x20U) 111 112 #define IWFSM_MAGICK 0x19cc7cc 113 #define IWFSM_CUSTOM_HDR_DATA_OFFSET \ 114 (4 /*magic*/ + 1 /*block pow*/ + 8 /*fsm bitmap block offset */ + 8 /*fsm bitmap block length*/ \ 115 + 8 /*all allocated block length sum */ + 4 /*number of all allocated areas */ \ 116 + 8 /* allocated areas length standard variance (deviation^2 * N) */ + 32 /*reserved*/ \ 117 + 4 /*custom hdr size*/) 118 119 /** File cleanup flags used in `IWFS_FSM::clear` */ 120 typedef uint8_t iwfs_fsm_clrfalgs; 121 122 /** Perform file size trimming after cleanup */ 123 #define IWFSM_CLEAR_TRIM ((iwfs_fsm_clrfalgs) 0x01U) 124 125 /** `IWFS_FSM` file open modes used in `IWFS_FSM_OPTS` */ 126 typedef uint8_t iwfs_fsm_openflags; 127 128 /** Do not use threading locks */ 129 #define IWFSM_NOLOCKS ((iwfs_fsm_openflags) 0x01U) 130 131 /** Strict block checking for alloc/dealloc operations. 10-15% performance overhead. */ 132 #define IWFSM_STRICT ((iwfs_fsm_openflags) 0x02U) 133 134 /** Do not trim fsm file on close */ 135 #define IWFSM_NO_TRIM_ON_CLOSE ((iwfs_fsm_openflags) 0x04U) 136 137 /** 138 * @brief Error codes specific to `IWFS_FSM`. 139 */ 140 typedef enum { 141 _IWFS_FSM_ERROR_START = (IW_ERROR_START + 4000UL), 142 IWFS_ERROR_NO_FREE_SPACE, /**< No free space. */ 143 IWFS_ERROR_INVALID_BLOCK_SIZE, /**< Invalid block size specified */ 144 IWFS_ERROR_RANGE_NOT_ALIGNED, 145 /**< Specified range/offset is not aligned with 146 page/block */ 147 IWFS_ERROR_FSM_SEGMENTATION, /**< Free-space map segmentation error */ 148 IWFS_ERROR_INVALID_FILEMETA, /**< Invalid file-metadata */ 149 IWFS_ERROR_PLATFORM_PAGE, 150 /**< Platform page size incopatibility, data 151 migration required. */ 152 IWFS_ERROR_RESIZE_FAIL, /**< Failed to resize file */ 153 _IWFS_FSM_ERROR_END, 154 } iwfs_fsm_ecode; 155 156 /** 157 * @brief `IWFS_FSM` file options. 158 * @see iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts) 159 */ 160 typedef struct IWFS_FSM_OPTS { 161 IWFS_EXT_OPTS exfile; 162 size_t bmlen; /**< Initial size of free-space bitmap */ 163 uint32_t hdrlen; /**< Length of custom file header.*/ 164 iwfs_fsm_openflags oflags; /**< Operation mode flags */ 165 iwfs_ext_mmap_opts_t mmap_opts; /**< Defaul mmap options used in `add_mmap` */ 166 uint8_t bpow; /**< Block size power for 2 */ 167 bool mmap_all; /**< Mmap all file data */ 168 } IWFS_FSM_OPTS; 169 170 /** 171 * @brief `IWFS_FSM` file state container. 172 * @see IWFS_FSM::state 173 */ 174 typedef struct IWFS_FSM_STATE { 175 IWFS_EXT_STATE exfile; /**< File pool state */ 176 size_t block_size; /**< Size of data block in bytes. */ 177 iwfs_fsm_openflags oflags; /**< Operation mode flags. */ 178 uint32_t hdrlen; /**< Length of custom file header length in bytes */ 179 uint64_t blocks_num; /**< Number of available data blocks. */ 180 uint64_t free_segments_num; /**< Number of free (deallocated) continuous data 181 segments. */ 182 double_t avg_alloc_size; /**< Average allocation number of blocks */ 183 double_t alloc_dispersion; /**< Average allocation blocks dispersion */ 184 } IWFS_FSM_STATE; 185 186 typedef struct IWFS_FSMDBG_STATE { 187 IWFS_FSM_STATE state; 188 uint64_t bmoff; 189 uint64_t bmlen; 190 uint64_t lfbklen; 191 uint64_t lfbkoff; 192 } IWFS_FSMDBG_STATE; 193 194 /** 195 * @brief Auto-expandable file with support of reader/writer address space 196 * locking 197 * and free space blocks management using bitmaps. 198 */ 199 typedef struct IWFS_FSM { 200 struct IWFS_FSM_IMPL *impl; 201 202 /** 203 * @brief Allocate a continuous address space within a file 204 * with length greater or equal to the desired @a len bytes. 205 * 206 * `Offset` and `length` allocated area will be block size aligned. 207 * 208 * @param f `IWFS_FSM` file. 209 * @param len Desired length of an allocated area in bytes. 210 * @param [in,out] oaddr Placeholder for the address of an allocated area. 211 * Value of @a oaddr passed to this function used as 212 * `hint` in order 213 * to allocate area located closely to the specified @a 214 * oaddr value. 215 * @param [out] olen Actual length of an allocated area in bytes. 216 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 217 * @return `0` on success or error code. 218 */ 219 iwrc (*allocate)( 220 struct IWFS_FSM *f, off_t len, off_t *oaddr, off_t *olen, 221 iwfs_fsm_aflags opts); 222 223 /** 224 * @brief Reallocate and adjust a size of an allocated block. 225 * 226 * If the given @a nlen value lesser than actual length of segment @a olen in 227 * that case 228 * segment will be truncated. 229 * 230 * @param f `IWFS_FSM` file. 231 * @param nlen Desired length of segment in bytes. 232 * @param oaddr [in,out] Address of an allocated segment. Placeholder for new 233 * address of reallocated segment. 234 * @param olen [in,out] Length of an allocated segment. Placeholder for length 235 * of reallocated segment. 236 * @param opts Allocation options bitmask flag @ref iwfs_fsm_aflags 237 * @return `0` on success or error code. 238 */ 239 iwrc (*reallocate)( 240 struct IWFS_FSM *f, off_t nlen, off_t *oaddr, off_t *olen, 241 iwfs_fsm_aflags opts); 242 243 /** 244 * @brief Free a previously allocated area. 245 * @param addr Address space offset in bytes <em>it must be block size 246 * aligned</em>. 247 * @param len Length of area to release. 248 * @return `0` on success or error code. 249 */ 250 iwrc (*deallocate)(struct IWFS_FSM *f, off_t addr, off_t len); 251 252 253 /** 254 * @brief Check allocation status of region specified by @a addr and @a len 255 * @return `0` on success or error code. 256 */ 257 iwrc (*check_allocation_status)(struct IWFS_FSM *f, off_t addr, off_t len, bool allocated); 258 259 /** 260 * @brief Write a data to the custom file header. 261 * 262 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 263 * file creation. 264 * 265 * @param off Offset position relative to custom header start offset. 266 * @param buf Data buffer to write 267 * @param siz Number of bytes of @a buf to write into header. 268 * @return `0` on success or error code. 269 */ 270 iwrc (*writehdr)(struct IWFS_FSM *f, off_t off, const void *buf, off_t siz); 271 272 /** 273 * @brief Read a data from the custom file header. 274 * 275 * A custom file header size specified in IWFS_FSM_OPTS::hdrlen options on 276 * file creation. 277 * 278 * @param off Offset position relative to custom header start offset. 279 * @param [out] buf Data buffer to read into 280 * @param Number of bytes to read 281 */ 282 iwrc (*readhdr)(struct IWFS_FSM *f, off_t off, void *buf, off_t siz); 283 284 /** 285 * @brief Cleanup all allocated data blocks and reset the file to the initial 286 * empty state. 287 * 288 * @param clrflags 289 * @return `0` on success or error code. 290 */ 291 iwrc (*clear)(struct IWFS_FSM *f, iwfs_fsm_clrfalgs clrflags); 292 293 /* See iwexfile.h */ 294 295 /** @see IWFS_EXT::ensure_size */ 296 iwrc (*ensure_size)(struct IWFS_FSM *f, off_t size); 297 298 299 /** @see IWFS_EXT::add_mmap */ 300 iwrc (*add_mmap)(struct IWFS_FSM *f, off_t off, size_t maxlen, iwfs_ext_mmap_opts_t opts); 301 302 303 /** @see IWFS_EXT::remap_all */ 304 iwrc (*remap_all)(struct IWFS_FSM *f); 305 306 /** 307 * @brief Get a pointer to the registered mmap area starting at `off`. 308 * 309 * WARNING: Internal read lock will be acquired and 310 * must be released by subsequent `release_mmap()` call 311 * after all activity with mmaped region has finished. 312 * 313 * @see IWFS_FSM::add_mmap 314 * @see IWFS_EXT::acquire_mmap 315 */ 316 iwrc (*acquire_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 317 318 /** 319 * @brief Retrieve mmaped region by its offset @a off 320 */ 321 iwrc (*probe_mmap)(struct IWFS_FSM *f, off_t off, uint8_t **mm, size_t *sp); 322 323 /** 324 * @brief Release the lock acquired by successfull call of `acquire_mmap()` 325 */ 326 iwrc (*release_mmap)(struct IWFS_FSM *f); 327 328 /** @see IWFS_EXT::remove_mmap */ 329 iwrc (*remove_mmap)(struct IWFS_FSM *f, off_t off); 330 331 /** @see IWFS_EXT::sync_mmap */ 332 iwrc (*sync_mmap)(struct IWFS_FSM *f, off_t off, iwfs_sync_flags flags); 333 334 /* See iwfile.h */ 335 336 /** @see IWFS_FILE::write */ 337 iwrc (*write)( 338 struct IWFS_FSM *f, off_t off, const void *buf, size_t siz, 339 size_t *sp); 340 341 /** @see IWFS_FILE::read */ 342 iwrc (*read)( 343 struct IWFS_FSM *f, off_t off, void *buf, size_t siz, 344 size_t *sp); 345 346 /** @see IWFS_FILE::close */ 347 iwrc (*close)(struct IWFS_FSM *f); 348 349 /** @see IWFS_FILE::sync */ 350 iwrc (*sync)(struct IWFS_FSM *f, iwfs_sync_flags flags); 351 352 /** @see IWFS_FILE::state */ 353 iwrc (*state)(struct IWFS_FSM *f, IWFS_FSM_STATE *state); 354 355 /** get access to the underlying iwextfile instance */ 356 iwrc (*extfile)(struct IWFS_FSM *f, IWFS_EXT **ext); 357 } IWFS_FSM; 358 359 /** 360 * @brief Open `IWFS_FSM` file. 361 * 362 * <strong>Example:</strong> 363 * 364 * Open a buffer pool file for multithreaded env with fibonacci file resize 365 * policy with block size of 64 bytes and custom file header of 255 bytes 366 * length. 367 * 368 * @code {.c} 369 * IWFS_FSM_OPTS opts = { 370 * .exfile = { 371 * .file = { 372 * .path = "myfile.dat", 373 * .omode = IWFS_OWRITE | IWFS_OCREATE, 374 * .lock_mode = IWP_WLOCK 375 * }, 376 * .rspolicy = iw_exfile_szpolicy_fibo 377 * }, 378 * .bpow = 6, // 2^6 bytes block size 379 * .hdrlen = 255, // Size of custom file header 380 * .oflags = IWFSM_STRICT // Use verbose free-space bitmap checking for 381 * // allocations (10-15% overhead) 382 * }; 383 * 384 * IWFS_FSM f; 385 * size_t sp; 386 * off_t space_len, space_addr = 0; 387 * 388 * iwrc rc = iwfs_fsmfile_open(&f, &opts); 389 * 390 * //Allocate 2 blocks of file space 391 * rc = f.allocate(&f, 128, &space_addr, &space_len, 0); 392 * if (!rc) { 393 * int data = 33; 394 * // Write some data to the allocated block with writer lock acquired on 395 * // `[space_addr, sizeof(data))` 396 * rc = f.lwrite(&f, space_addr, &data, sizeof(data), &sp); 397 * ... 398 * } 399 * ... 400 * @endcode 401 * 402 * @param f File handle 403 * @param opts File open options 404 * @relatesalso IWFS_FSM 405 */ 406 IW_EXPORT WUR iwrc iwfs_fsmfile_open(IWFS_FSM *f, const IWFS_FSM_OPTS *opts); 407 408 /** 409 * @brief Init `iwfsmfile` submodule. 410 */ 411 IW_EXPORT WUR iwrc iwfs_fsmfile_init(void); 412 413 IW_EXTERN_C_END 414 415 #endif 416