1 /// @file htslib/bgzf.h 2 /// Low-level routines for direct BGZF operations. 3 /* 4 Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology 5 2011, 2012 Attractive Chaos <attractor@live.co.uk> 6 Copyright (C) 2009, 2013, 2014,2017 Genome Research Ltd 7 8 Permission is hereby granted, free of charge, to any person obtaining a copy 9 of this software and associated documentation files (the "Software"), to deal 10 in the Software without restriction, including without limitation the rights 11 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 copies of the Software, and to permit persons to whom the Software is 13 furnished to do so, subject to the following conditions: 14 15 The above copyright notice and this permission notice shall be included in 16 all copies or substantial portions of the Software. 17 18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 THE SOFTWARE. 25 */ 26 27 /* The BGZF library was originally written by Bob Handsaker from the Broad 28 * Institute. It was later improved by the SAMtools developers. */ 29 30 #ifndef HTSLIB_BGZF_H 31 #define HTSLIB_BGZF_H 32 33 #include <stdint.h> 34 #include <stdio.h> 35 #include <zlib.h> 36 #include <sys/types.h> 37 38 #include "hts_defs.h" 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 #define BGZF_BLOCK_SIZE 0xff00 // make sure compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE 45 #define BGZF_MAX_BLOCK_SIZE 0x10000 46 47 #define BGZF_ERR_ZLIB 1 48 #define BGZF_ERR_HEADER 2 49 #define BGZF_ERR_IO 4 50 #define BGZF_ERR_MISUSE 8 51 #define BGZF_ERR_MT 16 // stream cannot be multi-threaded 52 #define BGZF_ERR_CRC 32 53 54 struct hFILE; 55 struct hts_tpool; 56 struct bgzf_mtaux_t; 57 typedef struct __bgzidx_t bgzidx_t; 58 typedef struct bgzf_cache_t bgzf_cache_t; 59 60 struct BGZF { 61 // Reserved bits should be written as 0; read as "don't care" 62 unsigned errcode:16, reserved:1, is_write:1, no_eof_block:1, is_be:1; 63 signed compress_level:9; 64 unsigned last_block_eof:1, is_compressed:1, is_gzip:1; 65 int cache_size; 66 int block_length, block_clength, block_offset; 67 int64_t block_address, uncompressed_address; 68 void *uncompressed_block, *compressed_block; 69 bgzf_cache_t *cache; 70 struct hFILE *fp; // actual file handle 71 struct bgzf_mtaux_t *mt; // only used for multi-threading 72 bgzidx_t *idx; // BGZF index 73 int idx_build_otf; // build index on the fly, set by bgzf_index_build_init() 74 z_stream *gz_stream;// for gzip-compressed files 75 }; 76 #ifndef HTS_BGZF_TYPEDEF 77 typedef struct BGZF BGZF; 78 #define HTS_BGZF_TYPEDEF 79 #endif 80 81 #ifndef KSTRING_T 82 #define KSTRING_T kstring_t 83 typedef struct __kstring_t { 84 size_t l, m; 85 char *s; 86 } kstring_t; 87 #endif 88 89 /****************** 90 * Basic routines * 91 ******************/ 92 93 /** 94 * Open an existing file descriptor for reading or writing. 95 * 96 * @param fd file descriptor 97 * Note that the file must be opened in binary mode, or else 98 * there will be problems on platforms that make a difference 99 * between text and binary mode. 100 * @param mode mode matching /[rwag][u0-9]+/: 'r' for reading, 'w' for 101 * writing, 'a' for appending, 'g' for gzip rather than BGZF 102 * compression (with 'w' only), and digit specifies the zlib 103 * compression level. 104 * Note that there is a distinction between 'u' and '0': the 105 * first yields plain uncompressed output whereas the latter 106 * outputs uncompressed data wrapped in the zlib format. 107 * @return BGZF file handler; 0 on error 108 */ 109 BGZF* bgzf_dopen(int fd, const char *mode); 110 111 #define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility 112 113 /** 114 * Open the specified file for reading or writing. 115 */ 116 BGZF* bgzf_open(const char* path, const char *mode); 117 118 /** 119 * Open an existing hFILE stream for reading or writing. 120 */ 121 BGZF* bgzf_hopen(struct hFILE *fp, const char *mode); 122 123 /** 124 * Close the BGZF and free all associated resources. 125 * 126 * @param fp BGZF file handler 127 * @return 0 on success and -1 on error 128 */ 129 int bgzf_close(BGZF *fp); 130 131 /** 132 * Read up to _length_ bytes from the file storing into _data_. 133 * 134 * @param fp BGZF file handler 135 * @param data data array to read into 136 * @param length size of data to read 137 * @return number of bytes actually read; 0 on end-of-file and -1 on error 138 */ 139 ssize_t bgzf_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED; 140 141 /** 142 * Write _length_ bytes from _data_ to the file. If no I/O errors occur, 143 * the complete _length_ bytes will be written (or queued for writing). 144 * 145 * @param fp BGZF file handler 146 * @param data data array to write 147 * @param length size of data to write 148 * @return number of bytes written (i.e., _length_); negative on error 149 */ 150 ssize_t bgzf_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED; 151 152 /** 153 * Write _length_ bytes from _data_ to the file, the index will be used to 154 * decide the amount of uncompressed data to be writen to each bgzip block. 155 * If no I/O errors occur, the complete _length_ bytes will be written (or 156 * queued for writing). 157 * @param fp BGZF file handler 158 * @param data data array to write 159 * @param length size of data to write 160 * @return number of bytes written (i.e., _length_); negative on error 161 */ 162 ssize_t bgzf_block_write(BGZF *fp, const void *data, size_t length); 163 164 /** 165 * Read up to _length_ bytes directly from the underlying stream without 166 * decompressing. Bypasses BGZF blocking, so must be used with care in 167 * specialised circumstances only. 168 * 169 * @param fp BGZF file handler 170 * @param data data array to read into 171 * @param length number of raw bytes to read 172 * @return number of bytes actually read; 0 on end-of-file and -1 on error 173 */ 174 ssize_t bgzf_raw_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED; 175 176 /** 177 * Write _length_ bytes directly to the underlying stream without 178 * compressing. Bypasses BGZF blocking, so must be used with care 179 * in specialised circumstances only. 180 * 181 * @param fp BGZF file handler 182 * @param data data array to write 183 * @param length number of raw bytes to write 184 * @return number of bytes actually written; -1 on error 185 */ 186 ssize_t bgzf_raw_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED; 187 188 /** 189 * Write the data in the buffer to the file. 190 * 191 * @param fp BGZF file handle 192 * @return 0 on success and -1 on error 193 */ 194 int bgzf_flush(BGZF *fp) HTS_RESULT_USED; 195 196 /** 197 * Return a virtual file pointer to the current location in the file. 198 * No interpretation of the value should be made, other than a subsequent 199 * call to bgzf_seek can be used to position the file at the same point. 200 * Return value is non-negative on success. 201 */ 202 #define bgzf_tell(fp) (((fp)->block_address << 16) | ((fp)->block_offset & 0xFFFF)) 203 204 /** 205 * Set the file to read from the location specified by _pos_. 206 * 207 * @param fp BGZF file handler 208 * @param pos virtual file offset returned by bgzf_tell() 209 * @param whence must be SEEK_SET 210 * @return 0 on success and -1 on error 211 */ 212 int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence) HTS_RESULT_USED; 213 214 /** 215 * Check if the BGZF end-of-file (EOF) marker is present 216 * 217 * @param fp BGZF file handler opened for reading 218 * @return 1 if the EOF marker is present and correct; 219 * 2 if it can't be checked, e.g., because fp isn't seekable; 220 * 0 if the EOF marker is absent; 221 * -1 (with errno set) on error 222 */ 223 int bgzf_check_EOF(BGZF *fp); 224 225 /** Return the file's compression format 226 * 227 * @param fp BGZF file handle 228 * @return A small integer matching the corresponding 229 * `enum htsCompression` value: 230 * - 0 / `no_compression` if the file is uncompressed 231 * - 1 / `gzip` if the file is plain GZIP-compressed 232 * - 2 / `bgzf` if the file is BGZF-compressed 233 * @since 1.4 234 */ 235 int bgzf_compression(BGZF *fp); 236 237 /** 238 * Check if a file is in the BGZF format 239 * 240 * @param fn file name 241 * @return 1 if _fn_ is BGZF; 0 if not or on I/O error 242 */ 243 int bgzf_is_bgzf(const char *fn) HTS_DEPRECATED("Use bgzf_compression() or hts_detect_format() instead"); 244 245 /********************* 246 * Advanced routines * 247 *********************/ 248 249 /** 250 * Set the cache size. Only effective when compiled with -DBGZF_CACHE. 251 * 252 * @param fp BGZF file handler 253 * @param size size of cache in bytes; 0 to disable caching (default) 254 */ 255 void bgzf_set_cache_size(BGZF *fp, int size); 256 257 /** 258 * Flush the file if the remaining buffer size is smaller than _size_ 259 * @return 0 if flushing succeeded or was not needed; negative on error 260 */ 261 int bgzf_flush_try(BGZF *fp, ssize_t size) HTS_RESULT_USED; 262 263 /** 264 * Read one byte from a BGZF file. It is faster than bgzf_read() 265 * @param fp BGZF file handler 266 * @return byte read; -1 on end-of-file or error 267 */ 268 int bgzf_getc(BGZF *fp); 269 270 /** 271 * Read one line from a BGZF file. It is faster than bgzf_getc() 272 * 273 * @param fp BGZF file handler 274 * @param delim delimitor 275 * @param str string to write to; must be initialized 276 * @return length of the string; -1 on end-of-file; <= -2 on error 277 */ 278 int bgzf_getline(BGZF *fp, int delim, kstring_t *str); 279 280 /** 281 * Read the next BGZF block. 282 */ 283 int bgzf_read_block(BGZF *fp) HTS_RESULT_USED; 284 285 /** 286 * Enable multi-threading (when compiled with -DBGZF_MT) via a shared 287 * thread pool. This means both encoder and decoder can balance 288 * usage across a single pool of worker jobs. 289 * 290 * @param fp BGZF file handler; must be opened for writing 291 * @param pool The thread pool (see hts_create_threads) 292 */ 293 int bgzf_thread_pool(BGZF *fp, struct hts_tpool *pool, int qsize); 294 295 /** 296 * Enable multi-threading (only effective when the library was compiled 297 * with -DBGZF_MT) 298 * 299 * @param fp BGZF file handler; must be opened for writing 300 * @param n_threads #threads used for writing 301 * @param n_sub_blks #blocks processed by each thread; a value 64-256 is recommended 302 */ 303 int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks); 304 305 /** 306 * Compress a single BGZF block. 307 * 308 * @param dst output buffer (must have size >= BGZF_MAX_BLOCK_SIZE) 309 * @param dlen size of output buffer; updated on return to the number 310 * of bytes actually written to dst 311 * @param src buffer to be compressed 312 * @param slen size of data to compress (must be <= BGZF_BLOCK_SIZE) 313 * @param level compression level 314 * @return 0 on success and negative on error 315 */ 316 int bgzf_compress(void *dst, size_t *dlen, const void *src, size_t slen, int level); 317 318 /******************* 319 * bgzidx routines * 320 *******************/ 321 322 /** 323 * Position BGZF at the uncompressed offset 324 * 325 * @param fp BGZF file handler; must be opened for reading 326 * @param uoffset file offset in the uncompressed data 327 * @param where SEEK_SET supported atm 328 * 329 * Returns 0 on success and -1 on error. 330 */ 331 int bgzf_useek(BGZF *fp, long uoffset, int where) HTS_RESULT_USED; 332 333 /** 334 * Position in uncompressed BGZF 335 * 336 * @param fp BGZF file handler; must be opened for reading 337 * 338 * Returns the current offset on success and -1 on error. 339 */ 340 long bgzf_utell(BGZF *fp); 341 342 /** 343 * Tell BGZF to build index while compressing. 344 * 345 * @param fp BGZF file handler; can be opened for reading or writing. 346 * 347 * Returns 0 on success and -1 on error. 348 */ 349 int bgzf_index_build_init(BGZF *fp); 350 351 /// Load BGZF index 352 /** 353 * @param fp BGZF file handler 354 * @param bname base name 355 * @param suffix suffix to add to bname (can be NULL) 356 * @return 0 on success and -1 on error. 357 */ 358 int bgzf_index_load(BGZF *fp, 359 const char *bname, const char *suffix) HTS_RESULT_USED; 360 361 /// Load BGZF index from an hFILE 362 /** 363 * @param fp BGZF file handle 364 * @param idx hFILE to read from 365 * @param name file name (for error reporting only; can be NULL) 366 * @return 0 on success and -1 on error. 367 * 368 * Populates @p fp with index data read from the hFILE handle @p idx. 369 * The file pointer to @idx should point to the start of the index 370 * data when this function is called. 371 * 372 * The file name can optionally be passed in the @p name parameter. This 373 * is only used for printing error messages; if NULL the word "index" is 374 * used instead. 375 */ 376 int bgzf_index_load_hfile(BGZF *fp, struct hFILE *idx, 377 const char *name) HTS_RESULT_USED; 378 379 /// Save BGZF index 380 /** 381 * @param fp BGZF file handler 382 * @param bname base name 383 * @param suffix suffix to add to bname (can be NULL) 384 * @return 0 on success and -1 on error. 385 */ 386 int bgzf_index_dump(BGZF *fp, 387 const char *bname, const char *suffix) HTS_RESULT_USED; 388 389 /// Write a BGZF index to an hFILE 390 /** 391 * @param fp BGZF file handle 392 * @param idx hFILE to write to 393 * @param name file name (for error reporting only, can be NULL) 394 * @return 0 on success and -1 on error. 395 * 396 * Write index data from @p fp to the file @p idx. 397 * 398 * The file name can optionally be passed in the @p name parameter. This 399 * is only used for printing error messages; if NULL the word "index" is 400 * used instead. 401 */ 402 403 int bgzf_index_dump_hfile(BGZF *fp, struct hFILE *idx, 404 const char *name) HTS_RESULT_USED; 405 406 #ifdef __cplusplus 407 } 408 #endif 409 410 #endif 411