1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2008 Oracle. All rights reserved. 4 * 5 * Based on jffs2 zlib code: 6 * Copyright © 2001-2007 Red Hat, Inc. 7 * Created by David Woodhouse <dwmw2@infradead.org> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/slab.h> 12 #include <linux/zlib.h> 13 #include <linux/zutil.h> 14 #include <linux/mm.h> 15 #include <linux/init.h> 16 #include <linux/err.h> 17 #include <linux/sched.h> 18 #include <linux/pagemap.h> 19 #include <linux/bio.h> 20 #include <linux/refcount.h> 21 #include "compression.h" 22 23 struct workspace { 24 z_stream strm; 25 char *buf; 26 struct list_head list; 27 int level; 28 }; 29 30 static struct workspace_manager wsm; 31 32 struct list_head *zlib_get_workspace(unsigned int level) 33 { 34 struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level); 35 struct workspace *workspace = list_entry(ws, struct workspace, list); 36 37 workspace->level = level; 38 39 return ws; 40 } 41 42 void zlib_free_workspace(struct list_head *ws) 43 { 44 struct workspace *workspace = list_entry(ws, struct workspace, list); 45 46 kvfree(workspace->strm.workspace); 47 kfree(workspace->buf); 48 kfree(workspace); 49 } 50 51 struct list_head *zlib_alloc_workspace(unsigned int level) 52 { 53 struct workspace *workspace; 54 int workspacesize; 55 56 workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); 57 if (!workspace) 58 return ERR_PTR(-ENOMEM); 59 60 workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), 61 zlib_inflate_workspacesize()); 62 workspace->strm.workspace = kvmalloc(workspacesize, GFP_KERNEL); 63 workspace->level = level; 64 workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 65 if (!workspace->strm.workspace || !workspace->buf) 66 goto fail; 67 68 INIT_LIST_HEAD(&workspace->list); 69 70 return &workspace->list; 71 fail: 72 zlib_free_workspace(&workspace->list); 73 return ERR_PTR(-ENOMEM); 74 } 75 76 int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, 77 u64 start, struct page **pages, unsigned long *out_pages, 78 unsigned long *total_in, unsigned long *total_out) 79 { 80 struct workspace *workspace = list_entry(ws, struct workspace, list); 81 int ret; 82 char *data_in; 83 char *cpage_out; 84 int nr_pages = 0; 85 struct page *in_page = NULL; 86 struct page *out_page = NULL; 87 unsigned long bytes_left; 88 unsigned long len = *total_out; 89 unsigned long nr_dest_pages = *out_pages; 90 const unsigned long max_out = nr_dest_pages * PAGE_SIZE; 91 92 *out_pages = 0; 93 *total_out = 0; 94 *total_in = 0; 95 96 if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) { 97 pr_warn("BTRFS: deflateInit failed\n"); 98 ret = -EIO; 99 goto out; 100 } 101 102 workspace->strm.total_in = 0; 103 workspace->strm.total_out = 0; 104 105 in_page = find_get_page(mapping, start >> PAGE_SHIFT); 106 data_in = kmap(in_page); 107 108 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 109 if (out_page == NULL) { 110 ret = -ENOMEM; 111 goto out; 112 } 113 cpage_out = kmap(out_page); 114 pages[0] = out_page; 115 nr_pages = 1; 116 117 workspace->strm.next_in = data_in; 118 workspace->strm.next_out = cpage_out; 119 workspace->strm.avail_out = PAGE_SIZE; 120 workspace->strm.avail_in = min(len, PAGE_SIZE); 121 122 while (workspace->strm.total_in < len) { 123 ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH); 124 if (ret != Z_OK) { 125 pr_debug("BTRFS: deflate in loop returned %d\n", 126 ret); 127 zlib_deflateEnd(&workspace->strm); 128 ret = -EIO; 129 goto out; 130 } 131 132 /* we're making it bigger, give up */ 133 if (workspace->strm.total_in > 8192 && 134 workspace->strm.total_in < 135 workspace->strm.total_out) { 136 ret = -E2BIG; 137 goto out; 138 } 139 /* we need another page for writing out. Test this 140 * before the total_in so we will pull in a new page for 141 * the stream end if required 142 */ 143 if (workspace->strm.avail_out == 0) { 144 kunmap(out_page); 145 if (nr_pages == nr_dest_pages) { 146 out_page = NULL; 147 ret = -E2BIG; 148 goto out; 149 } 150 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 151 if (out_page == NULL) { 152 ret = -ENOMEM; 153 goto out; 154 } 155 cpage_out = kmap(out_page); 156 pages[nr_pages] = out_page; 157 nr_pages++; 158 workspace->strm.avail_out = PAGE_SIZE; 159 workspace->strm.next_out = cpage_out; 160 } 161 /* we're all done */ 162 if (workspace->strm.total_in >= len) 163 break; 164 165 /* we've read in a full page, get a new one */ 166 if (workspace->strm.avail_in == 0) { 167 if (workspace->strm.total_out > max_out) 168 break; 169 170 bytes_left = len - workspace->strm.total_in; 171 kunmap(in_page); 172 put_page(in_page); 173 174 start += PAGE_SIZE; 175 in_page = find_get_page(mapping, 176 start >> PAGE_SHIFT); 177 data_in = kmap(in_page); 178 workspace->strm.avail_in = min(bytes_left, 179 PAGE_SIZE); 180 workspace->strm.next_in = data_in; 181 } 182 } 183 workspace->strm.avail_in = 0; 184 ret = zlib_deflate(&workspace->strm, Z_FINISH); 185 zlib_deflateEnd(&workspace->strm); 186 187 if (ret != Z_STREAM_END) { 188 ret = -EIO; 189 goto out; 190 } 191 192 if (workspace->strm.total_out >= workspace->strm.total_in) { 193 ret = -E2BIG; 194 goto out; 195 } 196 197 ret = 0; 198 *total_out = workspace->strm.total_out; 199 *total_in = workspace->strm.total_in; 200 out: 201 *out_pages = nr_pages; 202 if (out_page) 203 kunmap(out_page); 204 205 if (in_page) { 206 kunmap(in_page); 207 put_page(in_page); 208 } 209 return ret; 210 } 211 212 int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) 213 { 214 struct workspace *workspace = list_entry(ws, struct workspace, list); 215 int ret = 0, ret2; 216 int wbits = MAX_WBITS; 217 char *data_in; 218 size_t total_out = 0; 219 unsigned long page_in_index = 0; 220 size_t srclen = cb->compressed_len; 221 unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); 222 unsigned long buf_start; 223 struct page **pages_in = cb->compressed_pages; 224 u64 disk_start = cb->start; 225 struct bio *orig_bio = cb->orig_bio; 226 227 data_in = kmap(pages_in[page_in_index]); 228 workspace->strm.next_in = data_in; 229 workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE); 230 workspace->strm.total_in = 0; 231 232 workspace->strm.total_out = 0; 233 workspace->strm.next_out = workspace->buf; 234 workspace->strm.avail_out = PAGE_SIZE; 235 236 /* If it's deflate, and it's got no preset dictionary, then 237 we can tell zlib to skip the adler32 check. */ 238 if (srclen > 2 && !(data_in[1] & PRESET_DICT) && 239 ((data_in[0] & 0x0f) == Z_DEFLATED) && 240 !(((data_in[0]<<8) + data_in[1]) % 31)) { 241 242 wbits = -((data_in[0] >> 4) + 8); 243 workspace->strm.next_in += 2; 244 workspace->strm.avail_in -= 2; 245 } 246 247 if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) { 248 pr_warn("BTRFS: inflateInit failed\n"); 249 kunmap(pages_in[page_in_index]); 250 return -EIO; 251 } 252 while (workspace->strm.total_in < srclen) { 253 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH); 254 if (ret != Z_OK && ret != Z_STREAM_END) 255 break; 256 257 buf_start = total_out; 258 total_out = workspace->strm.total_out; 259 260 /* we didn't make progress in this inflate call, we're done */ 261 if (buf_start == total_out) 262 break; 263 264 ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start, 265 total_out, disk_start, 266 orig_bio); 267 if (ret2 == 0) { 268 ret = 0; 269 goto done; 270 } 271 272 workspace->strm.next_out = workspace->buf; 273 workspace->strm.avail_out = PAGE_SIZE; 274 275 if (workspace->strm.avail_in == 0) { 276 unsigned long tmp; 277 kunmap(pages_in[page_in_index]); 278 page_in_index++; 279 if (page_in_index >= total_pages_in) { 280 data_in = NULL; 281 break; 282 } 283 data_in = kmap(pages_in[page_in_index]); 284 workspace->strm.next_in = data_in; 285 tmp = srclen - workspace->strm.total_in; 286 workspace->strm.avail_in = min(tmp, 287 PAGE_SIZE); 288 } 289 } 290 if (ret != Z_STREAM_END) 291 ret = -EIO; 292 else 293 ret = 0; 294 done: 295 zlib_inflateEnd(&workspace->strm); 296 if (data_in) 297 kunmap(pages_in[page_in_index]); 298 if (!ret) 299 zero_fill_bio(orig_bio); 300 return ret; 301 } 302 303 int zlib_decompress(struct list_head *ws, unsigned char *data_in, 304 struct page *dest_page, unsigned long start_byte, size_t srclen, 305 size_t destlen) 306 { 307 struct workspace *workspace = list_entry(ws, struct workspace, list); 308 int ret = 0; 309 int wbits = MAX_WBITS; 310 unsigned long bytes_left; 311 unsigned long total_out = 0; 312 unsigned long pg_offset = 0; 313 char *kaddr; 314 315 destlen = min_t(unsigned long, destlen, PAGE_SIZE); 316 bytes_left = destlen; 317 318 workspace->strm.next_in = data_in; 319 workspace->strm.avail_in = srclen; 320 workspace->strm.total_in = 0; 321 322 workspace->strm.next_out = workspace->buf; 323 workspace->strm.avail_out = PAGE_SIZE; 324 workspace->strm.total_out = 0; 325 /* If it's deflate, and it's got no preset dictionary, then 326 we can tell zlib to skip the adler32 check. */ 327 if (srclen > 2 && !(data_in[1] & PRESET_DICT) && 328 ((data_in[0] & 0x0f) == Z_DEFLATED) && 329 !(((data_in[0]<<8) + data_in[1]) % 31)) { 330 331 wbits = -((data_in[0] >> 4) + 8); 332 workspace->strm.next_in += 2; 333 workspace->strm.avail_in -= 2; 334 } 335 336 if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) { 337 pr_warn("BTRFS: inflateInit failed\n"); 338 return -EIO; 339 } 340 341 while (bytes_left > 0) { 342 unsigned long buf_start; 343 unsigned long buf_offset; 344 unsigned long bytes; 345 346 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH); 347 if (ret != Z_OK && ret != Z_STREAM_END) 348 break; 349 350 buf_start = total_out; 351 total_out = workspace->strm.total_out; 352 353 if (total_out == buf_start) { 354 ret = -EIO; 355 break; 356 } 357 358 if (total_out <= start_byte) 359 goto next; 360 361 if (total_out > start_byte && buf_start < start_byte) 362 buf_offset = start_byte - buf_start; 363 else 364 buf_offset = 0; 365 366 bytes = min(PAGE_SIZE - pg_offset, 367 PAGE_SIZE - buf_offset); 368 bytes = min(bytes, bytes_left); 369 370 kaddr = kmap_atomic(dest_page); 371 memcpy(kaddr + pg_offset, workspace->buf + buf_offset, bytes); 372 kunmap_atomic(kaddr); 373 374 pg_offset += bytes; 375 bytes_left -= bytes; 376 next: 377 workspace->strm.next_out = workspace->buf; 378 workspace->strm.avail_out = PAGE_SIZE; 379 } 380 381 if (ret != Z_STREAM_END && bytes_left != 0) 382 ret = -EIO; 383 else 384 ret = 0; 385 386 zlib_inflateEnd(&workspace->strm); 387 388 /* 389 * this should only happen if zlib returned fewer bytes than we 390 * expected. btrfs_get_block is responsible for zeroing from the 391 * end of the inline extent (destlen) to the end of the page 392 */ 393 if (pg_offset < destlen) { 394 kaddr = kmap_atomic(dest_page); 395 memset(kaddr + pg_offset, 0, destlen - pg_offset); 396 kunmap_atomic(kaddr); 397 } 398 return ret; 399 } 400 401 const struct btrfs_compress_op btrfs_zlib_compress = { 402 .workspace_manager = &wsm, 403 .max_level = 9, 404 .default_level = BTRFS_ZLIB_DEFAULT_LEVEL, 405 }; 406