1 /* Copyright (c) Mark Harmstone 2016-17 2 * 3 * This file is part of WinBtrfs. 4 * 5 * WinBtrfs is free software: you can redistribute it and/or modify 6 * it under the terms of the GNU Lesser General Public Licence as published by 7 * the Free Software Foundation, either version 3 of the Licence, or 8 * (at your option) any later version. 9 * 10 * WinBtrfs is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU Lesser General Public Licence for more details. 14 * 15 * You should have received a copy of the GNU Lesser General Public Licence 16 * along with WinBtrfs. If not, see <http://www.gnu.org/licenses/>. */ 17 18 #include "btrfs_drv.h" 19 20 typedef struct { 21 uint64_t start; 22 uint64_t end; 23 uint8_t* data; 24 PMDL mdl; 25 uint64_t irp_offset; 26 } write_stripe; 27 28 _Function_class_(IO_COMPLETION_ROUTINE) 29 static NTSTATUS __stdcall write_data_completion(PDEVICE_OBJECT DeviceObject, PIRP Irp, PVOID conptr); 30 31 static void remove_fcb_extent(fcb* fcb, extent* ext, LIST_ENTRY* rollback); 32 33 extern tPsUpdateDiskCounters fPsUpdateDiskCounters; 34 extern tCcCopyWriteEx fCcCopyWriteEx; 35 extern tFsRtlUpdateDiskCounters fFsRtlUpdateDiskCounters; 36 extern bool diskacc; 37 38 bool find_data_address_in_chunk(device_extension* Vcb, chunk* c, uint64_t length, uint64_t* address) { 39 LIST_ENTRY* le; 40 space* s; 41 42 TRACE("(%p, %I64x, %I64x, %p)\n", Vcb, c->offset, length, address); 43 44 if (length > c->chunk_item->size - c->used) 45 return false; 46 47 if (!c->cache_loaded) { 48 NTSTATUS Status = load_cache_chunk(Vcb, c, NULL); 49 50 if (!NT_SUCCESS(Status)) { 51 ERR("load_cache_chunk returned %08x\n", Status); 52 return false; 53 } 54 } 55 56 if (IsListEmpty(&c->space_size)) 57 return false; 58 59 le = c->space_size.Flink; 60 while (le != &c->space_size) { 61 s = CONTAINING_RECORD(le, space, list_entry_size); 62 63 if (s->size == length) { 64 *address = s->address; 65 return true; 66 } else if (s->size < length) { 67 if (le == c->space_size.Flink) 68 return false; 69 70 s = CONTAINING_RECORD(le->Blink, space, list_entry_size); 71 72 *address = s->address; 73 return true; 74 } 75 76 le = le->Flink; 77 } 78 79 s = CONTAINING_RECORD(c->space_size.Blink, space, list_entry_size); 80 81 if (s->size > length) { 82 *address = s->address; 83 return true; 84 } 85 86 return false; 87 } 88 89 chunk* get_chunk_from_address(device_extension* Vcb, uint64_t address) { 90 LIST_ENTRY* le2; 91 92 ExAcquireResourceSharedLite(&Vcb->chunk_lock, true); 93 94 le2 = Vcb->chunks.Flink; 95 while (le2 != &Vcb->chunks) { 96 chunk* c = CONTAINING_RECORD(le2, chunk, list_entry); 97 98 if (address >= c->offset && address < c->offset + c->chunk_item->size) { 99 ExReleaseResourceLite(&Vcb->chunk_lock); 100 return c; 101 } 102 103 le2 = le2->Flink; 104 } 105 106 ExReleaseResourceLite(&Vcb->chunk_lock); 107 108 return NULL; 109 } 110 111 typedef struct { 112 space* dh; 113 device* device; 114 } stripe; 115 116 static uint64_t find_new_chunk_address(device_extension* Vcb, uint64_t size) { 117 uint64_t lastaddr; 118 LIST_ENTRY* le; 119 120 lastaddr = 0xc00000; 121 122 le = Vcb->chunks.Flink; 123 while (le != &Vcb->chunks) { 124 chunk* c = CONTAINING_RECORD(le, chunk, list_entry); 125 126 if (c->offset >= lastaddr + size) 127 return lastaddr; 128 129 lastaddr = c->offset + c->chunk_item->size; 130 131 le = le->Flink; 132 } 133 134 return lastaddr; 135 } 136 137 static bool find_new_dup_stripes(device_extension* Vcb, stripe* stripes, uint64_t max_stripe_size, bool full_size) { 138 uint64_t devusage = 0xffffffffffffffff; 139 space *devdh1 = NULL, *devdh2 = NULL; 140 LIST_ENTRY* le; 141 device* dev2 = NULL; 142 143 le = Vcb->devices.Flink; 144 145 while (le != &Vcb->devices) { 146 device* dev = CONTAINING_RECORD(le, device, list_entry); 147 148 if (!dev->readonly && !dev->reloc && dev->devobj) { 149 uint64_t usage = (dev->devitem.bytes_used * 4096) / dev->devitem.num_bytes; 150 151 // favour devices which have been used the least 152 if (usage < devusage) { 153 if (!IsListEmpty(&dev->space)) { 154 LIST_ENTRY* le2; 155 space *dh1 = NULL, *dh2 = NULL; 156 157 le2 = dev->space.Flink; 158 while (le2 != &dev->space) { 159 space* dh = CONTAINING_RECORD(le2, space, list_entry); 160 161 if (dh->size >= max_stripe_size && (!dh1 || !dh2 || dh->size < dh1->size)) { 162 dh2 = dh1; 163 dh1 = dh; 164 } 165 166 le2 = le2->Flink; 167 } 168 169 if (dh1 && (dh2 || dh1->size >= 2 * max_stripe_size)) { 170 dev2 = dev; 171 devusage = usage; 172 devdh1 = dh1; 173 devdh2 = dh2 ? dh2 : dh1; 174 } 175 } 176 } 177 } 178 179 le = le->Flink; 180 } 181 182 if (!devdh1) { 183 uint64_t size = 0; 184 185 // Can't find hole of at least max_stripe_size; look for the largest one we can find 186 187 if (full_size) 188 return false; 189 190 le = Vcb->devices.Flink; 191 while (le != &Vcb->devices) { 192 device* dev = CONTAINING_RECORD(le, device, list_entry); 193 194 if (!dev->readonly && !dev->reloc) { 195 if (!IsListEmpty(&dev->space)) { 196 LIST_ENTRY* le2; 197 space *dh1 = NULL, *dh2 = NULL; 198 199 le2 = dev->space.Flink; 200 while (le2 != &dev->space) { 201 space* dh = CONTAINING_RECORD(le2, space, list_entry); 202 203 if (!dh1 || !dh2 || dh->size < dh1->size) { 204 dh2 = dh1; 205 dh1 = dh; 206 } 207 208 le2 = le2->Flink; 209 } 210 211 if (dh1) { 212 uint64_t devsize; 213 214 if (dh2) 215 devsize = max(dh1->size / 2, min(dh1->size, dh2->size)); 216 else 217 devsize = dh1->size / 2; 218 219 if (devsize > size) { 220 dev2 = dev; 221 devdh1 = dh1; 222 223 if (dh2 && min(dh1->size, dh2->size) > dh1->size / 2) 224 devdh2 = dh2; 225 else 226 devdh2 = dh1; 227 228 size = devsize; 229 } 230 } 231 } 232 } 233 234 le = le->Flink; 235 } 236 237 if (!devdh1) 238 return false; 239 } 240 241 stripes[0].device = stripes[1].device = dev2; 242 stripes[0].dh = devdh1; 243 stripes[1].dh = devdh2; 244 245 return true; 246 } 247 248 static bool find_new_stripe(device_extension* Vcb, stripe* stripes, uint16_t i, uint64_t max_stripe_size, bool allow_missing, bool full_size) { 249 uint64_t k, devusage = 0xffffffffffffffff; 250 space* devdh = NULL; 251 LIST_ENTRY* le; 252 device* dev2 = NULL; 253 254 le = Vcb->devices.Flink; 255 while (le != &Vcb->devices) { 256 device* dev = CONTAINING_RECORD(le, device, list_entry); 257 uint64_t usage; 258 bool skip = false; 259 260 if (dev->readonly || dev->reloc || (!dev->devobj && !allow_missing)) { 261 le = le->Flink; 262 continue; 263 } 264 265 // skip this device if it already has a stripe 266 if (i > 0) { 267 for (k = 0; k < i; k++) { 268 if (stripes[k].device == dev) { 269 skip = true; 270 break; 271 } 272 } 273 } 274 275 if (!skip) { 276 usage = (dev->devitem.bytes_used * 4096) / dev->devitem.num_bytes; 277 278 // favour devices which have been used the least 279 if (usage < devusage) { 280 if (!IsListEmpty(&dev->space)) { 281 LIST_ENTRY* le2; 282 283 le2 = dev->space.Flink; 284 while (le2 != &dev->space) { 285 space* dh = CONTAINING_RECORD(le2, space, list_entry); 286 287 if ((dev2 != dev && dh->size >= max_stripe_size) || 288 (dev2 == dev && dh->size >= max_stripe_size && dh->size < devdh->size) 289 ) { 290 devdh = dh; 291 dev2 = dev; 292 devusage = usage; 293 } 294 295 le2 = le2->Flink; 296 } 297 } 298 } 299 } 300 301 le = le->Flink; 302 } 303 304 if (!devdh) { 305 // Can't find hole of at least max_stripe_size; look for the largest one we can find 306 307 if (full_size) 308 return false; 309 310 le = Vcb->devices.Flink; 311 while (le != &Vcb->devices) { 312 device* dev = CONTAINING_RECORD(le, device, list_entry); 313 bool skip = false; 314 315 if (dev->readonly || dev->reloc || (!dev->devobj && !allow_missing)) { 316 le = le->Flink; 317 continue; 318 } 319 320 // skip this device if it already has a stripe 321 if (i > 0) { 322 for (k = 0; k < i; k++) { 323 if (stripes[k].device == dev) { 324 skip = true; 325 break; 326 } 327 } 328 } 329 330 if (!skip) { 331 if (!IsListEmpty(&dev->space)) { 332 LIST_ENTRY* le2; 333 334 le2 = dev->space.Flink; 335 while (le2 != &dev->space) { 336 space* dh = CONTAINING_RECORD(le2, space, list_entry); 337 338 if (!devdh || devdh->size < dh->size) { 339 devdh = dh; 340 dev2 = dev; 341 } 342 343 le2 = le2->Flink; 344 } 345 } 346 } 347 348 le = le->Flink; 349 } 350 351 if (!devdh) 352 return false; 353 } 354 355 stripes[i].dh = devdh; 356 stripes[i].device = dev2; 357 358 return true; 359 } 360 361 NTSTATUS alloc_chunk(device_extension* Vcb, uint64_t flags, chunk** pc, bool full_size) { 362 NTSTATUS Status; 363 uint64_t max_stripe_size, max_chunk_size, stripe_size, stripe_length, factor; 364 uint64_t total_size = 0, logaddr; 365 uint16_t i, type, num_stripes, sub_stripes, max_stripes, min_stripes, allowed_missing; 366 stripe* stripes = NULL; 367 uint16_t cisize; 368 CHUNK_ITEM_STRIPE* cis; 369 chunk* c = NULL; 370 space* s = NULL; 371 LIST_ENTRY* le; 372 373 le = Vcb->devices.Flink; 374 while (le != &Vcb->devices) { 375 device* dev = CONTAINING_RECORD(le, device, list_entry); 376 total_size += dev->devitem.num_bytes; 377 378 le = le->Flink; 379 } 380 381 TRACE("total_size = %I64x\n", total_size); 382 383 // We purposely check for DATA first - mixed blocks have the same size 384 // as DATA ones. 385 if (flags & BLOCK_FLAG_DATA) { 386 max_stripe_size = 0x40000000; // 1 GB 387 max_chunk_size = 10 * max_stripe_size; 388 } else if (flags & BLOCK_FLAG_METADATA) { 389 if (total_size > 0xC80000000) // 50 GB 390 max_stripe_size = 0x40000000; // 1 GB 391 else 392 max_stripe_size = 0x10000000; // 256 MB 393 394 max_chunk_size = max_stripe_size; 395 } else if (flags & BLOCK_FLAG_SYSTEM) { 396 max_stripe_size = 0x2000000; // 32 MB 397 max_chunk_size = 2 * max_stripe_size; 398 } else { 399 ERR("unknown chunk type\n"); 400 return STATUS_INTERNAL_ERROR; 401 } 402 403 if (flags & BLOCK_FLAG_DUPLICATE) { 404 min_stripes = 2; 405 max_stripes = 2; 406 sub_stripes = 0; 407 type = BLOCK_FLAG_DUPLICATE; 408 allowed_missing = 0; 409 } else if (flags & BLOCK_FLAG_RAID0) { 410 min_stripes = 2; 411 max_stripes = (uint16_t)min(0xffff, Vcb->superblock.num_devices); 412 sub_stripes = 0; 413 type = BLOCK_FLAG_RAID0; 414 allowed_missing = 0; 415 } else if (flags & BLOCK_FLAG_RAID1) { 416 min_stripes = 2; 417 max_stripes = 2; 418 sub_stripes = 1; 419 type = BLOCK_FLAG_RAID1; 420 allowed_missing = 1; 421 } else if (flags & BLOCK_FLAG_RAID10) { 422 min_stripes = 4; 423 max_stripes = (uint16_t)min(0xffff, Vcb->superblock.num_devices); 424 sub_stripes = 2; 425 type = BLOCK_FLAG_RAID10; 426 allowed_missing = 1; 427 } else if (flags & BLOCK_FLAG_RAID5) { 428 min_stripes = 3; 429 max_stripes = (uint16_t)min(0xffff, Vcb->superblock.num_devices); 430 sub_stripes = 1; 431 type = BLOCK_FLAG_RAID5; 432 allowed_missing = 1; 433 } else if (flags & BLOCK_FLAG_RAID6) { 434 min_stripes = 4; 435 max_stripes = 257; 436 sub_stripes = 1; 437 type = BLOCK_FLAG_RAID6; 438 allowed_missing = 2; 439 } else { // SINGLE 440 min_stripes = 1; 441 max_stripes = 1; 442 sub_stripes = 1; 443 type = 0; 444 allowed_missing = 0; 445 } 446 447 if (max_chunk_size > total_size / 10) { // cap at 10% 448 max_chunk_size = total_size / 10; 449 max_stripe_size = max_chunk_size / min_stripes; 450 } 451 452 TRACE("would allocate a new chunk of %I64x bytes and stripe %I64x\n", max_chunk_size, max_stripe_size); 453 454 stripes = ExAllocatePoolWithTag(PagedPool, sizeof(stripe) * max_stripes, ALLOC_TAG); 455 if (!stripes) { 456 ERR("out of memory\n"); 457 Status = STATUS_INSUFFICIENT_RESOURCES; 458 goto end; 459 } 460 461 num_stripes = 0; 462 463 if (type == BLOCK_FLAG_DUPLICATE) { 464 if (!find_new_dup_stripes(Vcb, stripes, max_stripe_size, full_size)) { 465 Status = STATUS_DISK_FULL; 466 goto end; 467 } 468 else 469 num_stripes = max_stripes; 470 } else { 471 for (i = 0; i < max_stripes; i++) { 472 if (!find_new_stripe(Vcb, stripes, i, max_stripe_size, false, full_size)) 473 break; 474 else 475 num_stripes++; 476 } 477 } 478 479 if (num_stripes < min_stripes && Vcb->options.allow_degraded && allowed_missing > 0) { 480 uint16_t added_missing = 0; 481 482 for (i = num_stripes; i < max_stripes; i++) { 483 if (!find_new_stripe(Vcb, stripes, i, max_stripe_size, true, full_size)) 484 break; 485 else { 486 added_missing++; 487 if (added_missing >= allowed_missing) 488 break; 489 } 490 } 491 492 num_stripes += added_missing; 493 } 494 495 // for RAID10, round down to an even number of stripes 496 if (type == BLOCK_FLAG_RAID10 && (num_stripes % sub_stripes) != 0) { 497 num_stripes -= num_stripes % sub_stripes; 498 } 499 500 if (num_stripes < min_stripes) { 501 WARN("found %u stripes, needed at least %u\n", num_stripes, min_stripes); 502 Status = STATUS_DISK_FULL; 503 goto end; 504 } 505 506 c = ExAllocatePoolWithTag(NonPagedPool, sizeof(chunk), ALLOC_TAG); 507 if (!c) { 508 ERR("out of memory\n"); 509 Status = STATUS_INSUFFICIENT_RESOURCES; 510 goto end; 511 } 512 513 c->devices = NULL; 514 515 cisize = sizeof(CHUNK_ITEM) + (num_stripes * sizeof(CHUNK_ITEM_STRIPE)); 516 c->chunk_item = ExAllocatePoolWithTag(NonPagedPool, cisize, ALLOC_TAG); 517 if (!c->chunk_item) { 518 ERR("out of memory\n"); 519 Status = STATUS_INSUFFICIENT_RESOURCES; 520 goto end; 521 } 522 523 stripe_length = 0x10000; // FIXME? BTRFS_STRIPE_LEN in kernel 524 525 if (type == BLOCK_FLAG_DUPLICATE && stripes[1].dh == stripes[0].dh) 526 stripe_size = min(stripes[0].dh->size / 2, max_stripe_size); 527 else { 528 stripe_size = max_stripe_size; 529 for (i = 0; i < num_stripes; i++) { 530 if (stripes[i].dh->size < stripe_size) 531 stripe_size = stripes[i].dh->size; 532 } 533 } 534 535 if (type == 0 || type == BLOCK_FLAG_DUPLICATE || type == BLOCK_FLAG_RAID1) 536 factor = 1; 537 else if (type == BLOCK_FLAG_RAID0) 538 factor = num_stripes; 539 else if (type == BLOCK_FLAG_RAID10) 540 factor = num_stripes / sub_stripes; 541 else if (type == BLOCK_FLAG_RAID5) 542 factor = num_stripes - 1; 543 else if (type == BLOCK_FLAG_RAID6) 544 factor = num_stripes - 2; 545 546 if (stripe_size * factor > max_chunk_size) 547 stripe_size = max_chunk_size / factor; 548 549 if (stripe_size % stripe_length > 0) 550 stripe_size -= stripe_size % stripe_length; 551 552 if (stripe_size == 0) { 553 ERR("not enough free space found (stripe_size == 0)\n"); 554 Status = STATUS_DISK_FULL; 555 goto end; 556 } 557 558 c->chunk_item->size = stripe_size * factor; 559 c->chunk_item->root_id = Vcb->extent_root->id; 560 c->chunk_item->stripe_length = stripe_length; 561 c->chunk_item->type = flags; 562 c->chunk_item->opt_io_alignment = (uint32_t)c->chunk_item->stripe_length; 563 c->chunk_item->opt_io_width = (uint32_t)c->chunk_item->stripe_length; 564 c->chunk_item->sector_size = stripes[0].device->devitem.minimal_io_size; 565 c->chunk_item->num_stripes = num_stripes; 566 c->chunk_item->sub_stripes = sub_stripes; 567 568 c->devices = ExAllocatePoolWithTag(NonPagedPool, sizeof(device*) * num_stripes, ALLOC_TAG); 569 if (!c->devices) { 570 ERR("out of memory\n"); 571 Status = STATUS_INSUFFICIENT_RESOURCES; 572 goto end; 573 } 574 575 cis = (CHUNK_ITEM_STRIPE*)&c->chunk_item[1]; 576 for (i = 0; i < num_stripes; i++) { 577 cis[i].dev_id = stripes[i].device->devitem.dev_id; 578 579 if (type == BLOCK_FLAG_DUPLICATE && i == 1 && stripes[i].dh == stripes[0].dh) 580 cis[i].offset = stripes[0].dh->address + stripe_size; 581 else 582 cis[i].offset = stripes[i].dh->address; 583 584 cis[i].dev_uuid = stripes[i].device->devitem.device_uuid; 585 586 c->devices[i] = stripes[i].device; 587 } 588 589 logaddr = find_new_chunk_address(Vcb, c->chunk_item->size); 590 591 Vcb->superblock.chunk_root_generation = Vcb->superblock.generation; 592 593 c->size = cisize; 594 c->offset = logaddr; 595 c->used = c->oldused = 0; 596 c->cache = c->old_cache = NULL; 597 c->readonly = false; 598 c->reloc = false; 599 c->last_alloc_set = false; 600 c->last_stripe = 0; 601 c->cache_loaded = true; 602 c->changed = false; 603 c->space_changed = false; 604 c->balance_num = 0; 605 606 InitializeListHead(&c->space); 607 InitializeListHead(&c->space_size); 608 InitializeListHead(&c->deleting); 609 InitializeListHead(&c->changed_extents); 610 611 InitializeListHead(&c->range_locks); 612 ExInitializeResourceLite(&c->range_locks_lock); 613 KeInitializeEvent(&c->range_locks_event, NotificationEvent, false); 614 615 InitializeListHead(&c->partial_stripes); 616 ExInitializeResourceLite(&c->partial_stripes_lock); 617 618 ExInitializeResourceLite(&c->lock); 619 ExInitializeResourceLite(&c->changed_extents_lock); 620 621 s = ExAllocatePoolWithTag(NonPagedPool, sizeof(space), ALLOC_TAG); 622 if (!s) { 623 ERR("out of memory\n"); 624 Status = STATUS_INSUFFICIENT_RESOURCES; 625 goto end; 626 } 627 628 s->address = c->offset; 629 s->size = c->chunk_item->size; 630 InsertTailList(&c->space, &s->list_entry); 631 InsertTailList(&c->space_size, &s->list_entry_size); 632 633 protect_superblocks(c); 634 635 for (i = 0; i < num_stripes; i++) { 636 stripes[i].device->devitem.bytes_used += stripe_size; 637 638 space_list_subtract2(&stripes[i].device->space, NULL, cis[i].offset, stripe_size, NULL, NULL); 639 } 640 641 Status = STATUS_SUCCESS; 642 643 if (flags & BLOCK_FLAG_RAID5 || flags & BLOCK_FLAG_RAID6) 644 Vcb->superblock.incompat_flags |= BTRFS_INCOMPAT_FLAGS_RAID56; 645 646 end: 647 if (stripes) 648 ExFreePool(stripes); 649 650 if (!NT_SUCCESS(Status)) { 651 if (c) { 652 if (c->devices) 653 ExFreePool(c->devices); 654 655 if (c->chunk_item) 656 ExFreePool(c->chunk_item); 657 658 ExFreePool(c); 659 } 660 661 if (s) ExFreePool(s); 662 } else { 663 bool done = false; 664 665 le = Vcb->chunks.Flink; 666 while (le != &Vcb->chunks) { 667 chunk* c2 = CONTAINING_RECORD(le, chunk, list_entry); 668 669 if (c2->offset > c->offset) { 670 InsertHeadList(le->Blink, &c->list_entry); 671 done = true; 672 break; 673 } 674 675 le = le->Flink; 676 } 677 678 if (!done) 679 InsertTailList(&Vcb->chunks, &c->list_entry); 680 681 c->created = true; 682 c->changed = true; 683 c->space_changed = true; 684 c->list_entry_balance.Flink = NULL; 685 686 *pc = c; 687 } 688 689 return Status; 690 } 691 692 static NTSTATUS prepare_raid0_write(_Pre_satisfies_(_Curr_->chunk_item->num_stripes>0) _In_ chunk* c, _In_ uint64_t address, _In_reads_bytes_(length) void* data, 693 _In_ uint32_t length, _In_ write_stripe* stripes, _In_ PIRP Irp, _In_ uint64_t irp_offset, _In_ write_data_context* wtc) { 694 uint64_t startoff, endoff; 695 uint16_t startoffstripe, endoffstripe, stripenum; 696 uint64_t pos, *stripeoff; 697 uint32_t i; 698 bool file_write = Irp && Irp->MdlAddress && (Irp->MdlAddress->ByteOffset == 0); 699 PMDL master_mdl; 700 PFN_NUMBER* pfns; 701 702 stripeoff = ExAllocatePoolWithTag(PagedPool, sizeof(uint64_t) * c->chunk_item->num_stripes, ALLOC_TAG); 703 if (!stripeoff) { 704 ERR("out of memory\n"); 705 return STATUS_INSUFFICIENT_RESOURCES; 706 } 707 708 get_raid0_offset(address - c->offset, c->chunk_item->stripe_length, c->chunk_item->num_stripes, &startoff, &startoffstripe); 709 get_raid0_offset(address + length - c->offset - 1, c->chunk_item->stripe_length, c->chunk_item->num_stripes, &endoff, &endoffstripe); 710 711 if (file_write) { 712 master_mdl = Irp->MdlAddress; 713 714 pfns = (PFN_NUMBER*)(Irp->MdlAddress + 1); 715 pfns = &pfns[irp_offset >> PAGE_SHIFT]; 716 } else if (((ULONG_PTR)data % PAGE_SIZE) != 0) { 717 wtc->scratch = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG); 718 if (!wtc->scratch) { 719 ERR("out of memory\n"); 720 return STATUS_INSUFFICIENT_RESOURCES; 721 } 722 723 RtlCopyMemory(wtc->scratch, data, length); 724 725 master_mdl = IoAllocateMdl(wtc->scratch, length, false, false, NULL); 726 if (!master_mdl) { 727 ERR("out of memory\n"); 728 return STATUS_INSUFFICIENT_RESOURCES; 729 } 730 731 MmBuildMdlForNonPagedPool(master_mdl); 732 733 wtc->mdl = master_mdl; 734 735 pfns = (PFN_NUMBER*)(master_mdl + 1); 736 } else { 737 NTSTATUS Status = STATUS_SUCCESS; 738 739 master_mdl = IoAllocateMdl(data, length, false, false, NULL); 740 if (!master_mdl) { 741 ERR("out of memory\n"); 742 return STATUS_INSUFFICIENT_RESOURCES; 743 } 744 745 _SEH2_TRY { 746 MmProbeAndLockPages(master_mdl, KernelMode, IoReadAccess); 747 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 748 Status = _SEH2_GetExceptionCode(); 749 } _SEH2_END; 750 751 if (!NT_SUCCESS(Status)) { 752 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 753 IoFreeMdl(master_mdl); 754 return Status; 755 } 756 757 wtc->mdl = master_mdl; 758 759 pfns = (PFN_NUMBER*)(master_mdl + 1); 760 } 761 762 for (i = 0; i < c->chunk_item->num_stripes; i++) { 763 if (startoffstripe > i) 764 stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 765 else if (startoffstripe == i) 766 stripes[i].start = startoff; 767 else 768 stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length); 769 770 if (endoffstripe > i) 771 stripes[i].end = endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 772 else if (endoffstripe == i) 773 stripes[i].end = endoff + 1; 774 else 775 stripes[i].end = endoff - (endoff % c->chunk_item->stripe_length); 776 777 if (stripes[i].start != stripes[i].end) { 778 stripes[i].mdl = IoAllocateMdl(NULL, (ULONG)(stripes[i].end - stripes[i].start), false, false, NULL); 779 if (!stripes[i].mdl) { 780 ERR("IoAllocateMdl failed\n"); 781 ExFreePool(stripeoff); 782 return STATUS_INSUFFICIENT_RESOURCES; 783 } 784 } 785 } 786 787 pos = 0; 788 RtlZeroMemory(stripeoff, sizeof(uint64_t) * c->chunk_item->num_stripes); 789 790 stripenum = startoffstripe; 791 792 while (pos < length) { 793 PFN_NUMBER* stripe_pfns = (PFN_NUMBER*)(stripes[stripenum].mdl + 1); 794 795 if (pos == 0) { 796 uint32_t writelen = (uint32_t)min(stripes[stripenum].end - stripes[stripenum].start, 797 c->chunk_item->stripe_length - (stripes[stripenum].start % c->chunk_item->stripe_length)); 798 799 RtlCopyMemory(stripe_pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 800 801 stripeoff[stripenum] += writelen; 802 pos += writelen; 803 } else if (length - pos < c->chunk_item->stripe_length) { 804 RtlCopyMemory(&stripe_pfns[stripeoff[stripenum] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)((length - pos) * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 805 break; 806 } else { 807 RtlCopyMemory(&stripe_pfns[stripeoff[stripenum] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 808 809 stripeoff[stripenum] += c->chunk_item->stripe_length; 810 pos += c->chunk_item->stripe_length; 811 } 812 813 stripenum = (stripenum + 1) % c->chunk_item->num_stripes; 814 } 815 816 ExFreePool(stripeoff); 817 818 return STATUS_SUCCESS; 819 } 820 821 static NTSTATUS prepare_raid10_write(_Pre_satisfies_(_Curr_->chunk_item->sub_stripes>0&&_Curr_->chunk_item->num_stripes>=_Curr_->chunk_item->sub_stripes) _In_ chunk* c, 822 _In_ uint64_t address, _In_reads_bytes_(length) void* data, _In_ uint32_t length, _In_ write_stripe* stripes, 823 _In_ PIRP Irp, _In_ uint64_t irp_offset, _In_ write_data_context* wtc) { 824 uint64_t startoff, endoff; 825 uint16_t startoffstripe, endoffstripe, stripenum; 826 uint64_t pos, *stripeoff; 827 uint32_t i; 828 bool file_write = Irp && Irp->MdlAddress && (Irp->MdlAddress->ByteOffset == 0); 829 PMDL master_mdl; 830 PFN_NUMBER* pfns; 831 832 get_raid0_offset(address - c->offset, c->chunk_item->stripe_length, c->chunk_item->num_stripes / c->chunk_item->sub_stripes, &startoff, &startoffstripe); 833 get_raid0_offset(address + length - c->offset - 1, c->chunk_item->stripe_length, c->chunk_item->num_stripes / c->chunk_item->sub_stripes, &endoff, &endoffstripe); 834 835 stripenum = startoffstripe; 836 startoffstripe *= c->chunk_item->sub_stripes; 837 endoffstripe *= c->chunk_item->sub_stripes; 838 839 if (file_write) { 840 master_mdl = Irp->MdlAddress; 841 842 pfns = (PFN_NUMBER*)(Irp->MdlAddress + 1); 843 pfns = &pfns[irp_offset >> PAGE_SHIFT]; 844 } else if (((ULONG_PTR)data % PAGE_SIZE) != 0) { 845 wtc->scratch = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG); 846 if (!wtc->scratch) { 847 ERR("out of memory\n"); 848 return STATUS_INSUFFICIENT_RESOURCES; 849 } 850 851 RtlCopyMemory(wtc->scratch, data, length); 852 853 master_mdl = IoAllocateMdl(wtc->scratch, length, false, false, NULL); 854 if (!master_mdl) { 855 ERR("out of memory\n"); 856 return STATUS_INSUFFICIENT_RESOURCES; 857 } 858 859 MmBuildMdlForNonPagedPool(master_mdl); 860 861 wtc->mdl = master_mdl; 862 863 pfns = (PFN_NUMBER*)(master_mdl + 1); 864 } else { 865 NTSTATUS Status = STATUS_SUCCESS; 866 867 master_mdl = IoAllocateMdl(data, length, false, false, NULL); 868 if (!master_mdl) { 869 ERR("out of memory\n"); 870 return STATUS_INSUFFICIENT_RESOURCES; 871 } 872 873 _SEH2_TRY { 874 MmProbeAndLockPages(master_mdl, KernelMode, IoReadAccess); 875 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 876 Status = _SEH2_GetExceptionCode(); 877 } _SEH2_END; 878 879 if (!NT_SUCCESS(Status)) { 880 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 881 IoFreeMdl(master_mdl); 882 return Status; 883 } 884 885 wtc->mdl = master_mdl; 886 887 pfns = (PFN_NUMBER*)(master_mdl + 1); 888 } 889 890 for (i = 0; i < c->chunk_item->num_stripes; i += c->chunk_item->sub_stripes) { 891 uint16_t j; 892 893 if (startoffstripe > i) 894 stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 895 else if (startoffstripe == i) 896 stripes[i].start = startoff; 897 else 898 stripes[i].start = startoff - (startoff % c->chunk_item->stripe_length); 899 900 if (endoffstripe > i) 901 stripes[i].end = endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 902 else if (endoffstripe == i) 903 stripes[i].end = endoff + 1; 904 else 905 stripes[i].end = endoff - (endoff % c->chunk_item->stripe_length); 906 907 stripes[i].mdl = IoAllocateMdl(NULL, (ULONG)(stripes[i].end - stripes[i].start), false, false, NULL); 908 if (!stripes[i].mdl) { 909 ERR("IoAllocateMdl failed\n"); 910 return STATUS_INSUFFICIENT_RESOURCES; 911 } 912 913 for (j = 1; j < c->chunk_item->sub_stripes; j++) { 914 stripes[i+j].start = stripes[i].start; 915 stripes[i+j].end = stripes[i].end; 916 stripes[i+j].data = stripes[i].data; 917 stripes[i+j].mdl = stripes[i].mdl; 918 } 919 } 920 921 pos = 0; 922 923 stripeoff = ExAllocatePoolWithTag(PagedPool, sizeof(uint64_t) * c->chunk_item->num_stripes / c->chunk_item->sub_stripes, ALLOC_TAG); 924 if (!stripeoff) { 925 ERR("out of memory\n"); 926 return STATUS_INSUFFICIENT_RESOURCES; 927 } 928 929 RtlZeroMemory(stripeoff, sizeof(uint64_t) * c->chunk_item->num_stripes / c->chunk_item->sub_stripes); 930 931 while (pos < length) { 932 PFN_NUMBER* stripe_pfns = (PFN_NUMBER*)(stripes[stripenum * c->chunk_item->sub_stripes].mdl + 1); 933 934 if (pos == 0) { 935 uint32_t writelen = (uint32_t)min(stripes[stripenum * c->chunk_item->sub_stripes].end - stripes[stripenum * c->chunk_item->sub_stripes].start, 936 c->chunk_item->stripe_length - (stripes[stripenum * c->chunk_item->sub_stripes].start % c->chunk_item->stripe_length)); 937 938 RtlCopyMemory(stripe_pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 939 940 stripeoff[stripenum] += writelen; 941 pos += writelen; 942 } else if (length - pos < c->chunk_item->stripe_length) { 943 RtlCopyMemory(&stripe_pfns[stripeoff[stripenum] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)((length - pos) * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 944 break; 945 } else { 946 RtlCopyMemory(&stripe_pfns[stripeoff[stripenum] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 947 948 stripeoff[stripenum] += c->chunk_item->stripe_length; 949 pos += c->chunk_item->stripe_length; 950 } 951 952 stripenum = (stripenum + 1) % (c->chunk_item->num_stripes / c->chunk_item->sub_stripes); 953 } 954 955 ExFreePool(stripeoff); 956 957 return STATUS_SUCCESS; 958 } 959 960 static NTSTATUS add_partial_stripe(device_extension* Vcb, chunk *c, uint64_t address, uint32_t length, void* data) { 961 NTSTATUS Status; 962 LIST_ENTRY* le; 963 partial_stripe* ps; 964 uint64_t stripe_addr; 965 uint16_t num_data_stripes; 966 967 num_data_stripes = c->chunk_item->num_stripes - (c->chunk_item->type & BLOCK_FLAG_RAID5 ? 1 : 2); 968 stripe_addr = address - ((address - c->offset) % (num_data_stripes * c->chunk_item->stripe_length)); 969 970 ExAcquireResourceExclusiveLite(&c->partial_stripes_lock, true); 971 972 le = c->partial_stripes.Flink; 973 while (le != &c->partial_stripes) { 974 ps = CONTAINING_RECORD(le, partial_stripe, list_entry); 975 976 if (ps->address == stripe_addr) { 977 // update existing entry 978 979 RtlCopyMemory(ps->data + address - stripe_addr, data, length); 980 RtlClearBits(&ps->bmp, (ULONG)((address - stripe_addr) / Vcb->superblock.sector_size), length / Vcb->superblock.sector_size); 981 982 // if now filled, flush 983 if (RtlAreBitsClear(&ps->bmp, 0, (ULONG)((num_data_stripes * c->chunk_item->stripe_length) / Vcb->superblock.sector_size))) { 984 Status = flush_partial_stripe(Vcb, c, ps); 985 if (!NT_SUCCESS(Status)) { 986 ERR("flush_partial_stripe returned %08x\n", Status); 987 goto end; 988 } 989 990 RemoveEntryList(&ps->list_entry); 991 992 if (ps->bmparr) 993 ExFreePool(ps->bmparr); 994 995 ExFreePool(ps); 996 } 997 998 Status = STATUS_SUCCESS; 999 goto end; 1000 } else if (ps->address > stripe_addr) 1001 break; 1002 1003 le = le->Flink; 1004 } 1005 1006 // add new entry 1007 1008 ps = ExAllocatePoolWithTag(NonPagedPool, offsetof(partial_stripe, data[0]) + (ULONG)(num_data_stripes * c->chunk_item->stripe_length), ALLOC_TAG); 1009 if (!ps) { 1010 ERR("out of memory\n"); 1011 Status = STATUS_INSUFFICIENT_RESOURCES; 1012 goto end; 1013 } 1014 1015 ps->bmplen = (ULONG)(num_data_stripes * c->chunk_item->stripe_length) / Vcb->superblock.sector_size; 1016 1017 ps->address = stripe_addr; 1018 ps->bmparr = ExAllocatePoolWithTag(NonPagedPool, (size_t)sector_align(((ps->bmplen / 8) + 1), sizeof(ULONG)), ALLOC_TAG); 1019 if (!ps->bmparr) { 1020 ERR("out of memory\n"); 1021 ExFreePool(ps); 1022 Status = STATUS_INSUFFICIENT_RESOURCES; 1023 goto end; 1024 } 1025 1026 RtlInitializeBitMap(&ps->bmp, ps->bmparr, ps->bmplen); 1027 RtlSetAllBits(&ps->bmp); 1028 1029 RtlCopyMemory(ps->data + address - stripe_addr, data, length); 1030 RtlClearBits(&ps->bmp, (ULONG)((address - stripe_addr) / Vcb->superblock.sector_size), length / Vcb->superblock.sector_size); 1031 1032 InsertHeadList(le->Blink, &ps->list_entry); 1033 1034 Status = STATUS_SUCCESS; 1035 1036 end: 1037 ExReleaseResourceLite(&c->partial_stripes_lock); 1038 1039 return Status; 1040 } 1041 1042 typedef struct { 1043 PMDL mdl; 1044 PFN_NUMBER* pfns; 1045 } log_stripe; 1046 1047 static NTSTATUS prepare_raid5_write(device_extension* Vcb, chunk* c, uint64_t address, void* data, uint32_t length, write_stripe* stripes, PIRP Irp, 1048 uint64_t irp_offset, ULONG priority, write_data_context* wtc) { 1049 uint64_t startoff, endoff, parity_start, parity_end; 1050 uint16_t startoffstripe, endoffstripe, parity, num_data_stripes = c->chunk_item->num_stripes - 1; 1051 uint64_t pos, parity_pos, *stripeoff = NULL; 1052 uint32_t i; 1053 bool file_write = Irp && Irp->MdlAddress && (Irp->MdlAddress->ByteOffset == 0); 1054 PMDL master_mdl; 1055 NTSTATUS Status; 1056 PFN_NUMBER *pfns, *parity_pfns; 1057 log_stripe* log_stripes = NULL; 1058 1059 if ((address + length - c->offset) % (num_data_stripes * c->chunk_item->stripe_length) > 0) { 1060 uint64_t delta = (address + length - c->offset) % (num_data_stripes * c->chunk_item->stripe_length); 1061 1062 delta = min(irp_offset + length, delta); 1063 Status = add_partial_stripe(Vcb, c, address + length - delta, (uint32_t)delta, (uint8_t*)data + irp_offset + length - delta); 1064 if (!NT_SUCCESS(Status)) { 1065 ERR("add_partial_stripe returned %08x\n", Status); 1066 goto exit; 1067 } 1068 1069 length -= (uint32_t)delta; 1070 } 1071 1072 if (length > 0 && (address - c->offset) % (num_data_stripes * c->chunk_item->stripe_length) > 0) { 1073 uint64_t delta = (num_data_stripes * c->chunk_item->stripe_length) - ((address - c->offset) % (num_data_stripes * c->chunk_item->stripe_length)); 1074 1075 Status = add_partial_stripe(Vcb, c, address, (uint32_t)delta, (uint8_t*)data + irp_offset); 1076 if (!NT_SUCCESS(Status)) { 1077 ERR("add_partial_stripe returned %08x\n", Status); 1078 goto exit; 1079 } 1080 1081 address += delta; 1082 length -= (uint32_t)delta; 1083 irp_offset += delta; 1084 } 1085 1086 if (length == 0) { 1087 Status = STATUS_SUCCESS; 1088 goto exit; 1089 } 1090 1091 get_raid0_offset(address - c->offset, c->chunk_item->stripe_length, num_data_stripes, &startoff, &startoffstripe); 1092 get_raid0_offset(address + length - c->offset - 1, c->chunk_item->stripe_length, num_data_stripes, &endoff, &endoffstripe); 1093 1094 pos = 0; 1095 while (pos < length) { 1096 parity = (((address - c->offset + pos) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1097 1098 if (pos == 0) { 1099 uint16_t stripe = (parity + startoffstripe + 1) % c->chunk_item->num_stripes; 1100 ULONG skip, writelen; 1101 1102 i = startoffstripe; 1103 while (stripe != parity) { 1104 if (i == startoffstripe) { 1105 writelen = (ULONG)min(length, c->chunk_item->stripe_length - (startoff % c->chunk_item->stripe_length)); 1106 1107 stripes[stripe].start = startoff; 1108 stripes[stripe].end = startoff + writelen; 1109 1110 pos += writelen; 1111 1112 if (pos == length) 1113 break; 1114 } else { 1115 writelen = (ULONG)min(length - pos, c->chunk_item->stripe_length); 1116 1117 stripes[stripe].start = startoff - (startoff % c->chunk_item->stripe_length); 1118 stripes[stripe].end = stripes[stripe].start + writelen; 1119 1120 pos += writelen; 1121 1122 if (pos == length) 1123 break; 1124 } 1125 1126 i++; 1127 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1128 } 1129 1130 if (pos == length) 1131 break; 1132 1133 for (i = 0; i < startoffstripe; i++) { 1134 stripe = (parity + i + 1) % c->chunk_item->num_stripes; 1135 1136 stripes[stripe].start = stripes[stripe].end = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1137 } 1138 1139 stripes[parity].start = stripes[parity].end = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1140 1141 if (length - pos > c->chunk_item->num_stripes * num_data_stripes * c->chunk_item->stripe_length) { 1142 skip = (ULONG)(((length - pos) / (c->chunk_item->num_stripes * num_data_stripes * c->chunk_item->stripe_length)) - 1); 1143 1144 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1145 stripes[i].end += skip * c->chunk_item->num_stripes * c->chunk_item->stripe_length; 1146 } 1147 1148 pos += skip * num_data_stripes * c->chunk_item->num_stripes * c->chunk_item->stripe_length; 1149 } 1150 } else if (length - pos >= c->chunk_item->stripe_length * num_data_stripes) { 1151 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1152 stripes[i].end += c->chunk_item->stripe_length; 1153 } 1154 1155 pos += c->chunk_item->stripe_length * num_data_stripes; 1156 } else { 1157 uint16_t stripe = (parity + 1) % c->chunk_item->num_stripes; 1158 1159 i = 0; 1160 while (stripe != parity) { 1161 if (endoffstripe == i) { 1162 stripes[stripe].end = endoff + 1; 1163 break; 1164 } else if (endoffstripe > i) 1165 stripes[stripe].end = endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1166 1167 i++; 1168 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1169 } 1170 1171 break; 1172 } 1173 } 1174 1175 parity_start = 0xffffffffffffffff; 1176 parity_end = 0; 1177 1178 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1179 if (stripes[i].start != 0 || stripes[i].end != 0) { 1180 parity_start = min(stripes[i].start, parity_start); 1181 parity_end = max(stripes[i].end, parity_end); 1182 } 1183 } 1184 1185 if (parity_end == parity_start) { 1186 Status = STATUS_SUCCESS; 1187 goto exit; 1188 } 1189 1190 parity = (((address - c->offset) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1191 stripes[parity].start = parity_start; 1192 1193 parity = (((address - c->offset + length - 1) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1194 stripes[parity].end = parity_end; 1195 1196 log_stripes = ExAllocatePoolWithTag(NonPagedPool, sizeof(log_stripe) * num_data_stripes, ALLOC_TAG); 1197 if (!log_stripes) { 1198 ERR("out of memory\n"); 1199 Status = STATUS_INSUFFICIENT_RESOURCES; 1200 goto exit; 1201 } 1202 1203 RtlZeroMemory(log_stripes, sizeof(log_stripe) * num_data_stripes); 1204 1205 for (i = 0; i < num_data_stripes; i++) { 1206 log_stripes[i].mdl = IoAllocateMdl(NULL, (ULONG)(parity_end - parity_start), false, false, NULL); 1207 if (!log_stripes[i].mdl) { 1208 ERR("out of memory\n"); 1209 Status = STATUS_INSUFFICIENT_RESOURCES; 1210 goto exit; 1211 } 1212 1213 log_stripes[i].mdl->MdlFlags |= MDL_PARTIAL; 1214 log_stripes[i].pfns = (PFN_NUMBER*)(log_stripes[i].mdl + 1); 1215 } 1216 1217 wtc->parity1 = ExAllocatePoolWithTag(NonPagedPool, (ULONG)(parity_end - parity_start), ALLOC_TAG); 1218 if (!wtc->parity1) { 1219 ERR("out of memory\n"); 1220 Status = STATUS_INSUFFICIENT_RESOURCES; 1221 goto exit; 1222 } 1223 1224 wtc->parity1_mdl = IoAllocateMdl(wtc->parity1, (ULONG)(parity_end - parity_start), false, false, NULL); 1225 if (!wtc->parity1_mdl) { 1226 ERR("out of memory\n"); 1227 Status = STATUS_INSUFFICIENT_RESOURCES; 1228 goto exit; 1229 } 1230 1231 MmBuildMdlForNonPagedPool(wtc->parity1_mdl); 1232 1233 if (file_write) 1234 master_mdl = Irp->MdlAddress; 1235 else if (((ULONG_PTR)data % PAGE_SIZE) != 0) { 1236 wtc->scratch = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG); 1237 if (!wtc->scratch) { 1238 ERR("out of memory\n"); 1239 Status = STATUS_INSUFFICIENT_RESOURCES; 1240 goto exit; 1241 } 1242 1243 RtlCopyMemory(wtc->scratch, (uint8_t*)data + irp_offset, length); 1244 1245 master_mdl = IoAllocateMdl(wtc->scratch, length, false, false, NULL); 1246 if (!master_mdl) { 1247 ERR("out of memory\n"); 1248 Status = STATUS_INSUFFICIENT_RESOURCES; 1249 goto exit; 1250 } 1251 1252 MmBuildMdlForNonPagedPool(master_mdl); 1253 1254 wtc->mdl = master_mdl; 1255 } else { 1256 master_mdl = IoAllocateMdl((uint8_t*)data + irp_offset, length, false, false, NULL); 1257 if (!master_mdl) { 1258 ERR("out of memory\n"); 1259 Status = STATUS_INSUFFICIENT_RESOURCES; 1260 goto exit; 1261 } 1262 1263 Status = STATUS_SUCCESS; 1264 1265 _SEH2_TRY { 1266 MmProbeAndLockPages(master_mdl, KernelMode, IoReadAccess); 1267 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 1268 Status = _SEH2_GetExceptionCode(); 1269 } _SEH2_END; 1270 1271 if (!NT_SUCCESS(Status)) { 1272 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 1273 IoFreeMdl(master_mdl); 1274 return Status; 1275 } 1276 1277 wtc->mdl = master_mdl; 1278 } 1279 1280 pfns = (PFN_NUMBER*)(master_mdl + 1); 1281 parity_pfns = (PFN_NUMBER*)(wtc->parity1_mdl + 1); 1282 1283 if (file_write) 1284 pfns = &pfns[irp_offset >> PAGE_SHIFT]; 1285 1286 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1287 if (stripes[i].start != stripes[i].end) { 1288 stripes[i].mdl = IoAllocateMdl((uint8_t*)MmGetMdlVirtualAddress(master_mdl) + irp_offset, (ULONG)(stripes[i].end - stripes[i].start), false, false, NULL); 1289 if (!stripes[i].mdl) { 1290 ERR("IoAllocateMdl failed\n"); 1291 Status = STATUS_INSUFFICIENT_RESOURCES; 1292 goto exit; 1293 } 1294 } 1295 } 1296 1297 stripeoff = ExAllocatePoolWithTag(PagedPool, sizeof(uint64_t) * c->chunk_item->num_stripes, ALLOC_TAG); 1298 if (!stripeoff) { 1299 ERR("out of memory\n"); 1300 Status = STATUS_INSUFFICIENT_RESOURCES; 1301 goto exit; 1302 } 1303 1304 RtlZeroMemory(stripeoff, sizeof(uint64_t) * c->chunk_item->num_stripes); 1305 1306 pos = 0; 1307 parity_pos = 0; 1308 1309 while (pos < length) { 1310 PFN_NUMBER* stripe_pfns; 1311 1312 parity = (((address - c->offset + pos) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1313 1314 if (pos == 0) { 1315 uint16_t stripe = (parity + startoffstripe + 1) % c->chunk_item->num_stripes; 1316 uint32_t writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, 1317 c->chunk_item->stripe_length - (stripes[stripe].start % c->chunk_item->stripe_length))); 1318 uint32_t maxwritelen = writelen; 1319 1320 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1321 1322 RtlCopyMemory(stripe_pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1323 1324 RtlCopyMemory(log_stripes[startoffstripe].pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1325 log_stripes[startoffstripe].pfns += writelen >> PAGE_SHIFT; 1326 1327 stripeoff[stripe] = writelen; 1328 pos += writelen; 1329 1330 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1331 i = startoffstripe + 1; 1332 1333 while (stripe != parity) { 1334 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1335 writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, c->chunk_item->stripe_length)); 1336 1337 if (writelen == 0) 1338 break; 1339 1340 if (writelen > maxwritelen) 1341 maxwritelen = writelen; 1342 1343 RtlCopyMemory(stripe_pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1344 1345 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1346 log_stripes[i].pfns += writelen >> PAGE_SHIFT; 1347 1348 stripeoff[stripe] = writelen; 1349 pos += writelen; 1350 1351 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1352 i++; 1353 } 1354 1355 stripe_pfns = (PFN_NUMBER*)(stripes[parity].mdl + 1); 1356 1357 RtlCopyMemory(stripe_pfns, parity_pfns, maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1358 stripeoff[parity] = maxwritelen; 1359 parity_pos = maxwritelen; 1360 } else if (length - pos >= c->chunk_item->stripe_length * num_data_stripes) { 1361 uint16_t stripe = (parity + 1) % c->chunk_item->num_stripes; 1362 1363 i = 0; 1364 while (stripe != parity) { 1365 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1366 1367 RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1368 1369 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1370 log_stripes[i].pfns += c->chunk_item->stripe_length >> PAGE_SHIFT; 1371 1372 stripeoff[stripe] += c->chunk_item->stripe_length; 1373 pos += c->chunk_item->stripe_length; 1374 1375 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1376 i++; 1377 } 1378 1379 stripe_pfns = (PFN_NUMBER*)(stripes[parity].mdl + 1); 1380 1381 RtlCopyMemory(&stripe_pfns[stripeoff[parity] >> PAGE_SHIFT], &parity_pfns[parity_pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1382 stripeoff[parity] += c->chunk_item->stripe_length; 1383 parity_pos += c->chunk_item->stripe_length; 1384 } else { 1385 uint16_t stripe = (parity + 1) % c->chunk_item->num_stripes; 1386 uint32_t writelen, maxwritelen = 0; 1387 1388 i = 0; 1389 while (pos < length) { 1390 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1391 writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, c->chunk_item->stripe_length)); 1392 1393 if (writelen == 0) 1394 break; 1395 1396 if (writelen > maxwritelen) 1397 maxwritelen = writelen; 1398 1399 RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1400 1401 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1402 log_stripes[i].pfns += writelen >> PAGE_SHIFT; 1403 1404 stripeoff[stripe] += writelen; 1405 pos += writelen; 1406 1407 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1408 i++; 1409 } 1410 1411 stripe_pfns = (PFN_NUMBER*)(stripes[parity].mdl + 1); 1412 1413 RtlCopyMemory(&stripe_pfns[stripeoff[parity] >> PAGE_SHIFT], &parity_pfns[parity_pos >> PAGE_SHIFT], maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1414 } 1415 } 1416 1417 for (i = 0; i < num_data_stripes; i++) { 1418 uint8_t* ss = MmGetSystemAddressForMdlSafe(log_stripes[i].mdl, priority); 1419 1420 if (i == 0) 1421 RtlCopyMemory(wtc->parity1, ss, (uint32_t)(parity_end - parity_start)); 1422 else 1423 do_xor(wtc->parity1, ss, (uint32_t)(parity_end - parity_start)); 1424 } 1425 1426 Status = STATUS_SUCCESS; 1427 1428 exit: 1429 if (log_stripes) { 1430 for (i = 0; i < num_data_stripes; i++) { 1431 if (log_stripes[i].mdl) 1432 IoFreeMdl(log_stripes[i].mdl); 1433 } 1434 1435 ExFreePool(log_stripes); 1436 } 1437 1438 if (stripeoff) 1439 ExFreePool(stripeoff); 1440 1441 return Status; 1442 } 1443 1444 static NTSTATUS prepare_raid6_write(device_extension* Vcb, chunk* c, uint64_t address, void* data, uint32_t length, write_stripe* stripes, PIRP Irp, 1445 uint64_t irp_offset, ULONG priority, write_data_context* wtc) { 1446 uint64_t startoff, endoff, parity_start, parity_end; 1447 uint16_t startoffstripe, endoffstripe, parity1, num_data_stripes = c->chunk_item->num_stripes - 2; 1448 uint64_t pos, parity_pos, *stripeoff = NULL; 1449 uint32_t i; 1450 bool file_write = Irp && Irp->MdlAddress && (Irp->MdlAddress->ByteOffset == 0); 1451 PMDL master_mdl; 1452 NTSTATUS Status; 1453 PFN_NUMBER *pfns, *parity1_pfns, *parity2_pfns; 1454 log_stripe* log_stripes = NULL; 1455 1456 if ((address + length - c->offset) % (num_data_stripes * c->chunk_item->stripe_length) > 0) { 1457 uint64_t delta = (address + length - c->offset) % (num_data_stripes * c->chunk_item->stripe_length); 1458 1459 delta = min(irp_offset + length, delta); 1460 Status = add_partial_stripe(Vcb, c, address + length - delta, (uint32_t)delta, (uint8_t*)data + irp_offset + length - delta); 1461 if (!NT_SUCCESS(Status)) { 1462 ERR("add_partial_stripe returned %08x\n", Status); 1463 goto exit; 1464 } 1465 1466 length -= (uint32_t)delta; 1467 } 1468 1469 if (length > 0 && (address - c->offset) % (num_data_stripes * c->chunk_item->stripe_length) > 0) { 1470 uint64_t delta = (num_data_stripes * c->chunk_item->stripe_length) - ((address - c->offset) % (num_data_stripes * c->chunk_item->stripe_length)); 1471 1472 Status = add_partial_stripe(Vcb, c, address, (uint32_t)delta, (uint8_t*)data + irp_offset); 1473 if (!NT_SUCCESS(Status)) { 1474 ERR("add_partial_stripe returned %08x\n", Status); 1475 goto exit; 1476 } 1477 1478 address += delta; 1479 length -= (uint32_t)delta; 1480 irp_offset += delta; 1481 } 1482 1483 if (length == 0) { 1484 Status = STATUS_SUCCESS; 1485 goto exit; 1486 } 1487 1488 get_raid0_offset(address - c->offset, c->chunk_item->stripe_length, num_data_stripes, &startoff, &startoffstripe); 1489 get_raid0_offset(address + length - c->offset - 1, c->chunk_item->stripe_length, num_data_stripes, &endoff, &endoffstripe); 1490 1491 pos = 0; 1492 while (pos < length) { 1493 parity1 = (((address - c->offset + pos) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1494 1495 if (pos == 0) { 1496 uint16_t stripe = (parity1 + startoffstripe + 2) % c->chunk_item->num_stripes; 1497 uint16_t parity2 = (parity1 + 1) % c->chunk_item->num_stripes; 1498 ULONG skip, writelen; 1499 1500 i = startoffstripe; 1501 while (stripe != parity1) { 1502 if (i == startoffstripe) { 1503 writelen = (ULONG)min(length, c->chunk_item->stripe_length - (startoff % c->chunk_item->stripe_length)); 1504 1505 stripes[stripe].start = startoff; 1506 stripes[stripe].end = startoff + writelen; 1507 1508 pos += writelen; 1509 1510 if (pos == length) 1511 break; 1512 } else { 1513 writelen = (ULONG)min(length - pos, c->chunk_item->stripe_length); 1514 1515 stripes[stripe].start = startoff - (startoff % c->chunk_item->stripe_length); 1516 stripes[stripe].end = stripes[stripe].start + writelen; 1517 1518 pos += writelen; 1519 1520 if (pos == length) 1521 break; 1522 } 1523 1524 i++; 1525 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1526 } 1527 1528 if (pos == length) 1529 break; 1530 1531 for (i = 0; i < startoffstripe; i++) { 1532 stripe = (parity1 + i + 2) % c->chunk_item->num_stripes; 1533 1534 stripes[stripe].start = stripes[stripe].end = startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1535 } 1536 1537 stripes[parity1].start = stripes[parity1].end = stripes[parity2].start = stripes[parity2].end = 1538 startoff - (startoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1539 1540 if (length - pos > c->chunk_item->num_stripes * num_data_stripes * c->chunk_item->stripe_length) { 1541 skip = (ULONG)(((length - pos) / (c->chunk_item->num_stripes * num_data_stripes * c->chunk_item->stripe_length)) - 1); 1542 1543 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1544 stripes[i].end += skip * c->chunk_item->num_stripes * c->chunk_item->stripe_length; 1545 } 1546 1547 pos += skip * num_data_stripes * c->chunk_item->num_stripes * c->chunk_item->stripe_length; 1548 } 1549 } else if (length - pos >= c->chunk_item->stripe_length * num_data_stripes) { 1550 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1551 stripes[i].end += c->chunk_item->stripe_length; 1552 } 1553 1554 pos += c->chunk_item->stripe_length * num_data_stripes; 1555 } else { 1556 uint16_t stripe = (parity1 + 2) % c->chunk_item->num_stripes; 1557 1558 i = 0; 1559 while (stripe != parity1) { 1560 if (endoffstripe == i) { 1561 stripes[stripe].end = endoff + 1; 1562 break; 1563 } else if (endoffstripe > i) 1564 stripes[stripe].end = endoff - (endoff % c->chunk_item->stripe_length) + c->chunk_item->stripe_length; 1565 1566 i++; 1567 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1568 } 1569 1570 break; 1571 } 1572 } 1573 1574 parity_start = 0xffffffffffffffff; 1575 parity_end = 0; 1576 1577 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1578 if (stripes[i].start != 0 || stripes[i].end != 0) { 1579 parity_start = min(stripes[i].start, parity_start); 1580 parity_end = max(stripes[i].end, parity_end); 1581 } 1582 } 1583 1584 if (parity_end == parity_start) { 1585 Status = STATUS_SUCCESS; 1586 goto exit; 1587 } 1588 1589 parity1 = (((address - c->offset) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1590 stripes[parity1].start = stripes[(parity1 + 1) % c->chunk_item->num_stripes].start = parity_start; 1591 1592 parity1 = (((address - c->offset + length - 1) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1593 stripes[parity1].end = stripes[(parity1 + 1) % c->chunk_item->num_stripes].end = parity_end; 1594 1595 log_stripes = ExAllocatePoolWithTag(NonPagedPool, sizeof(log_stripe) * num_data_stripes, ALLOC_TAG); 1596 if (!log_stripes) { 1597 ERR("out of memory\n"); 1598 Status = STATUS_INSUFFICIENT_RESOURCES; 1599 goto exit; 1600 } 1601 1602 RtlZeroMemory(log_stripes, sizeof(log_stripe) * num_data_stripes); 1603 1604 for (i = 0; i < num_data_stripes; i++) { 1605 log_stripes[i].mdl = IoAllocateMdl(NULL, (ULONG)(parity_end - parity_start), false, false, NULL); 1606 if (!log_stripes[i].mdl) { 1607 ERR("out of memory\n"); 1608 Status = STATUS_INSUFFICIENT_RESOURCES; 1609 goto exit; 1610 } 1611 1612 log_stripes[i].mdl->MdlFlags |= MDL_PARTIAL; 1613 log_stripes[i].pfns = (PFN_NUMBER*)(log_stripes[i].mdl + 1); 1614 } 1615 1616 wtc->parity1 = ExAllocatePoolWithTag(NonPagedPool, (ULONG)(parity_end - parity_start), ALLOC_TAG); 1617 if (!wtc->parity1) { 1618 ERR("out of memory\n"); 1619 Status = STATUS_INSUFFICIENT_RESOURCES; 1620 goto exit; 1621 } 1622 1623 wtc->parity2 = ExAllocatePoolWithTag(NonPagedPool, (ULONG)(parity_end - parity_start), ALLOC_TAG); 1624 if (!wtc->parity2) { 1625 ERR("out of memory\n"); 1626 Status = STATUS_INSUFFICIENT_RESOURCES; 1627 goto exit; 1628 } 1629 1630 wtc->parity1_mdl = IoAllocateMdl(wtc->parity1, (ULONG)(parity_end - parity_start), false, false, NULL); 1631 if (!wtc->parity1_mdl) { 1632 ERR("out of memory\n"); 1633 Status = STATUS_INSUFFICIENT_RESOURCES; 1634 goto exit; 1635 } 1636 1637 MmBuildMdlForNonPagedPool(wtc->parity1_mdl); 1638 1639 wtc->parity2_mdl = IoAllocateMdl(wtc->parity2, (ULONG)(parity_end - parity_start), false, false, NULL); 1640 if (!wtc->parity2_mdl) { 1641 ERR("out of memory\n"); 1642 Status = STATUS_INSUFFICIENT_RESOURCES; 1643 goto exit; 1644 } 1645 1646 MmBuildMdlForNonPagedPool(wtc->parity2_mdl); 1647 1648 if (file_write) 1649 master_mdl = Irp->MdlAddress; 1650 else if (((ULONG_PTR)data % PAGE_SIZE) != 0) { 1651 wtc->scratch = ExAllocatePoolWithTag(NonPagedPool, length, ALLOC_TAG); 1652 if (!wtc->scratch) { 1653 ERR("out of memory\n"); 1654 Status = STATUS_INSUFFICIENT_RESOURCES; 1655 goto exit; 1656 } 1657 1658 RtlCopyMemory(wtc->scratch, (uint8_t*)data + irp_offset, length); 1659 1660 master_mdl = IoAllocateMdl(wtc->scratch, length, false, false, NULL); 1661 if (!master_mdl) { 1662 ERR("out of memory\n"); 1663 Status = STATUS_INSUFFICIENT_RESOURCES; 1664 goto exit; 1665 } 1666 1667 MmBuildMdlForNonPagedPool(master_mdl); 1668 1669 wtc->mdl = master_mdl; 1670 } else { 1671 master_mdl = IoAllocateMdl((uint8_t*)data + irp_offset, length, false, false, NULL); 1672 if (!master_mdl) { 1673 ERR("out of memory\n"); 1674 Status = STATUS_INSUFFICIENT_RESOURCES; 1675 goto exit; 1676 } 1677 1678 Status = STATUS_SUCCESS; 1679 1680 _SEH2_TRY { 1681 MmProbeAndLockPages(master_mdl, KernelMode, IoReadAccess); 1682 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 1683 Status = _SEH2_GetExceptionCode(); 1684 } _SEH2_END; 1685 1686 if (!NT_SUCCESS(Status)) { 1687 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 1688 IoFreeMdl(master_mdl); 1689 goto exit; 1690 } 1691 1692 wtc->mdl = master_mdl; 1693 } 1694 1695 pfns = (PFN_NUMBER*)(master_mdl + 1); 1696 parity1_pfns = (PFN_NUMBER*)(wtc->parity1_mdl + 1); 1697 parity2_pfns = (PFN_NUMBER*)(wtc->parity2_mdl + 1); 1698 1699 if (file_write) 1700 pfns = &pfns[irp_offset >> PAGE_SHIFT]; 1701 1702 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1703 if (stripes[i].start != stripes[i].end) { 1704 stripes[i].mdl = IoAllocateMdl((uint8_t*)MmGetMdlVirtualAddress(master_mdl) + irp_offset, (ULONG)(stripes[i].end - stripes[i].start), false, false, NULL); 1705 if (!stripes[i].mdl) { 1706 ERR("IoAllocateMdl failed\n"); 1707 Status = STATUS_INSUFFICIENT_RESOURCES; 1708 goto exit; 1709 } 1710 } 1711 } 1712 1713 stripeoff = ExAllocatePoolWithTag(PagedPool, sizeof(uint64_t) * c->chunk_item->num_stripes, ALLOC_TAG); 1714 if (!stripeoff) { 1715 ERR("out of memory\n"); 1716 Status = STATUS_INSUFFICIENT_RESOURCES; 1717 goto exit; 1718 } 1719 1720 RtlZeroMemory(stripeoff, sizeof(uint64_t) * c->chunk_item->num_stripes); 1721 1722 pos = 0; 1723 parity_pos = 0; 1724 1725 while (pos < length) { 1726 PFN_NUMBER* stripe_pfns; 1727 1728 parity1 = (((address - c->offset + pos) / (num_data_stripes * c->chunk_item->stripe_length)) + num_data_stripes) % c->chunk_item->num_stripes; 1729 1730 if (pos == 0) { 1731 uint16_t stripe = (parity1 + startoffstripe + 2) % c->chunk_item->num_stripes, parity2; 1732 uint32_t writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, 1733 c->chunk_item->stripe_length - (stripes[stripe].start % c->chunk_item->stripe_length))); 1734 uint32_t maxwritelen = writelen; 1735 1736 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1737 1738 RtlCopyMemory(stripe_pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1739 1740 RtlCopyMemory(log_stripes[startoffstripe].pfns, pfns, writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1741 log_stripes[startoffstripe].pfns += writelen >> PAGE_SHIFT; 1742 1743 stripeoff[stripe] = writelen; 1744 pos += writelen; 1745 1746 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1747 i = startoffstripe + 1; 1748 1749 while (stripe != parity1) { 1750 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1751 writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, c->chunk_item->stripe_length)); 1752 1753 if (writelen == 0) 1754 break; 1755 1756 if (writelen > maxwritelen) 1757 maxwritelen = writelen; 1758 1759 RtlCopyMemory(stripe_pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1760 1761 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1762 log_stripes[i].pfns += writelen >> PAGE_SHIFT; 1763 1764 stripeoff[stripe] = writelen; 1765 pos += writelen; 1766 1767 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1768 i++; 1769 } 1770 1771 stripe_pfns = (PFN_NUMBER*)(stripes[parity1].mdl + 1); 1772 RtlCopyMemory(stripe_pfns, parity1_pfns, maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1773 stripeoff[parity1] = maxwritelen; 1774 1775 parity2 = (parity1 + 1) % c->chunk_item->num_stripes; 1776 1777 stripe_pfns = (PFN_NUMBER*)(stripes[parity2].mdl + 1); 1778 RtlCopyMemory(stripe_pfns, parity2_pfns, maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1779 stripeoff[parity2] = maxwritelen; 1780 1781 parity_pos = maxwritelen; 1782 } else if (length - pos >= c->chunk_item->stripe_length * num_data_stripes) { 1783 uint16_t stripe = (parity1 + 2) % c->chunk_item->num_stripes, parity2; 1784 1785 i = 0; 1786 while (stripe != parity1) { 1787 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1788 1789 RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1790 1791 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1792 log_stripes[i].pfns += c->chunk_item->stripe_length >> PAGE_SHIFT; 1793 1794 stripeoff[stripe] += c->chunk_item->stripe_length; 1795 pos += c->chunk_item->stripe_length; 1796 1797 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1798 i++; 1799 } 1800 1801 stripe_pfns = (PFN_NUMBER*)(stripes[parity1].mdl + 1); 1802 RtlCopyMemory(&stripe_pfns[stripeoff[parity1] >> PAGE_SHIFT], &parity1_pfns[parity_pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1803 stripeoff[parity1] += c->chunk_item->stripe_length; 1804 1805 parity2 = (parity1 + 1) % c->chunk_item->num_stripes; 1806 1807 stripe_pfns = (PFN_NUMBER*)(stripes[parity2].mdl + 1); 1808 RtlCopyMemory(&stripe_pfns[stripeoff[parity2] >> PAGE_SHIFT], &parity2_pfns[parity_pos >> PAGE_SHIFT], (ULONG)(c->chunk_item->stripe_length * sizeof(PFN_NUMBER) >> PAGE_SHIFT)); 1809 stripeoff[parity2] += c->chunk_item->stripe_length; 1810 1811 parity_pos += c->chunk_item->stripe_length; 1812 } else { 1813 uint16_t stripe = (parity1 + 2) % c->chunk_item->num_stripes, parity2; 1814 uint32_t writelen, maxwritelen = 0; 1815 1816 i = 0; 1817 while (pos < length) { 1818 stripe_pfns = (PFN_NUMBER*)(stripes[stripe].mdl + 1); 1819 writelen = (uint32_t)min(length - pos, min(stripes[stripe].end - stripes[stripe].start, c->chunk_item->stripe_length)); 1820 1821 if (writelen == 0) 1822 break; 1823 1824 if (writelen > maxwritelen) 1825 maxwritelen = writelen; 1826 1827 RtlCopyMemory(&stripe_pfns[stripeoff[stripe] >> PAGE_SHIFT], &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1828 1829 RtlCopyMemory(log_stripes[i].pfns, &pfns[pos >> PAGE_SHIFT], writelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1830 log_stripes[i].pfns += writelen >> PAGE_SHIFT; 1831 1832 stripeoff[stripe] += writelen; 1833 pos += writelen; 1834 1835 stripe = (stripe + 1) % c->chunk_item->num_stripes; 1836 i++; 1837 } 1838 1839 stripe_pfns = (PFN_NUMBER*)(stripes[parity1].mdl + 1); 1840 RtlCopyMemory(&stripe_pfns[stripeoff[parity1] >> PAGE_SHIFT], &parity1_pfns[parity_pos >> PAGE_SHIFT], maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1841 1842 parity2 = (parity1 + 1) % c->chunk_item->num_stripes; 1843 1844 stripe_pfns = (PFN_NUMBER*)(stripes[parity2].mdl + 1); 1845 RtlCopyMemory(&stripe_pfns[stripeoff[parity2] >> PAGE_SHIFT], &parity2_pfns[parity_pos >> PAGE_SHIFT], maxwritelen * sizeof(PFN_NUMBER) >> PAGE_SHIFT); 1846 } 1847 } 1848 1849 for (i = 0; i < num_data_stripes; i++) { 1850 uint8_t* ss = MmGetSystemAddressForMdlSafe(log_stripes[c->chunk_item->num_stripes - 3 - i].mdl, priority); 1851 1852 if (i == 0) { 1853 RtlCopyMemory(wtc->parity1, ss, (ULONG)(parity_end - parity_start)); 1854 RtlCopyMemory(wtc->parity2, ss, (ULONG)(parity_end - parity_start)); 1855 } else { 1856 do_xor(wtc->parity1, ss, (uint32_t)(parity_end - parity_start)); 1857 1858 galois_double(wtc->parity2, (uint32_t)(parity_end - parity_start)); 1859 do_xor(wtc->parity2, ss, (uint32_t)(parity_end - parity_start)); 1860 } 1861 } 1862 1863 Status = STATUS_SUCCESS; 1864 1865 exit: 1866 if (log_stripes) { 1867 for (i = 0; i < num_data_stripes; i++) { 1868 if (log_stripes[i].mdl) 1869 IoFreeMdl(log_stripes[i].mdl); 1870 } 1871 1872 ExFreePool(log_stripes); 1873 } 1874 1875 if (stripeoff) 1876 ExFreePool(stripeoff); 1877 1878 return Status; 1879 } 1880 1881 NTSTATUS write_data(_In_ device_extension* Vcb, _In_ uint64_t address, _In_reads_bytes_(length) void* data, _In_ uint32_t length, _In_ write_data_context* wtc, 1882 _In_opt_ PIRP Irp, _In_opt_ chunk* c, _In_ bool file_write, _In_ uint64_t irp_offset, _In_ ULONG priority) { 1883 NTSTATUS Status; 1884 uint32_t i; 1885 CHUNK_ITEM_STRIPE* cis; 1886 write_stripe* stripes = NULL; 1887 uint64_t total_writing = 0; 1888 ULONG allowed_missing, missing; 1889 1890 TRACE("(%p, %I64x, %p, %x)\n", Vcb, address, data, length); 1891 1892 if (!c) { 1893 c = get_chunk_from_address(Vcb, address); 1894 if (!c) { 1895 ERR("could not get chunk for address %I64x\n", address); 1896 return STATUS_INTERNAL_ERROR; 1897 } 1898 } 1899 1900 stripes = ExAllocatePoolWithTag(PagedPool, sizeof(write_stripe) * c->chunk_item->num_stripes, ALLOC_TAG); 1901 if (!stripes) { 1902 ERR("out of memory\n"); 1903 return STATUS_INSUFFICIENT_RESOURCES; 1904 } 1905 1906 RtlZeroMemory(stripes, sizeof(write_stripe) * c->chunk_item->num_stripes); 1907 1908 cis = (CHUNK_ITEM_STRIPE*)&c->chunk_item[1]; 1909 1910 if (c->chunk_item->type & BLOCK_FLAG_RAID0) { 1911 Status = prepare_raid0_write(c, address, data, length, stripes, file_write ? Irp : NULL, irp_offset, wtc); 1912 if (!NT_SUCCESS(Status)) { 1913 ERR("prepare_raid0_write returned %08x\n", Status); 1914 goto prepare_failed; 1915 } 1916 1917 allowed_missing = 0; 1918 } else if (c->chunk_item->type & BLOCK_FLAG_RAID10) { 1919 Status = prepare_raid10_write(c, address, data, length, stripes, file_write ? Irp : NULL, irp_offset, wtc); 1920 if (!NT_SUCCESS(Status)) { 1921 ERR("prepare_raid10_write returned %08x\n", Status); 1922 goto prepare_failed; 1923 } 1924 1925 allowed_missing = 1; 1926 } else if (c->chunk_item->type & BLOCK_FLAG_RAID5) { 1927 Status = prepare_raid5_write(Vcb, c, address, data, length, stripes, file_write ? Irp : NULL, irp_offset, priority, wtc); 1928 if (!NT_SUCCESS(Status)) { 1929 ERR("prepare_raid5_write returned %08x\n", Status); 1930 goto prepare_failed; 1931 } 1932 1933 allowed_missing = 1; 1934 } else if (c->chunk_item->type & BLOCK_FLAG_RAID6) { 1935 Status = prepare_raid6_write(Vcb, c, address, data, length, stripes, file_write ? Irp : NULL, irp_offset, priority, wtc); 1936 if (!NT_SUCCESS(Status)) { 1937 ERR("prepare_raid6_write returned %08x\n", Status); 1938 goto prepare_failed; 1939 } 1940 1941 allowed_missing = 2; 1942 } else { // write same data to every location - SINGLE, DUP, RAID1 1943 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1944 stripes[i].start = address - c->offset; 1945 stripes[i].end = stripes[i].start + length; 1946 stripes[i].data = data; 1947 stripes[i].irp_offset = irp_offset; 1948 1949 if (c->devices[i]->devobj) { 1950 if (file_write) { 1951 uint8_t* va; 1952 ULONG writelen = (ULONG)(stripes[i].end - stripes[i].start); 1953 1954 va = (uint8_t*)MmGetMdlVirtualAddress(Irp->MdlAddress) + stripes[i].irp_offset; 1955 1956 stripes[i].mdl = IoAllocateMdl(va, writelen, false, false, NULL); 1957 if (!stripes[i].mdl) { 1958 ERR("IoAllocateMdl failed\n"); 1959 Status = STATUS_INSUFFICIENT_RESOURCES; 1960 goto prepare_failed; 1961 } 1962 1963 IoBuildPartialMdl(Irp->MdlAddress, stripes[i].mdl, va, writelen); 1964 } else { 1965 stripes[i].mdl = IoAllocateMdl(stripes[i].data, (ULONG)(stripes[i].end - stripes[i].start), false, false, NULL); 1966 if (!stripes[i].mdl) { 1967 ERR("IoAllocateMdl failed\n"); 1968 Status = STATUS_INSUFFICIENT_RESOURCES; 1969 goto prepare_failed; 1970 } 1971 1972 Status = STATUS_SUCCESS; 1973 1974 _SEH2_TRY { 1975 MmProbeAndLockPages(stripes[i].mdl, KernelMode, IoReadAccess); 1976 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 1977 Status = _SEH2_GetExceptionCode(); 1978 } _SEH2_END; 1979 1980 if (!NT_SUCCESS(Status)) { 1981 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 1982 IoFreeMdl(stripes[i].mdl); 1983 stripes[i].mdl = NULL; 1984 goto prepare_failed; 1985 } 1986 } 1987 } 1988 } 1989 1990 allowed_missing = c->chunk_item->num_stripes - 1; 1991 } 1992 1993 missing = 0; 1994 for (i = 0; i < c->chunk_item->num_stripes; i++) { 1995 if (!c->devices[i]->devobj) 1996 missing++; 1997 } 1998 1999 if (missing > allowed_missing) { 2000 ERR("cannot write as %u missing devices (maximum %u)\n", missing, allowed_missing); 2001 Status = STATUS_DEVICE_NOT_READY; 2002 goto prepare_failed; 2003 } 2004 2005 for (i = 0; i < c->chunk_item->num_stripes; i++) { 2006 write_data_stripe* stripe; 2007 PIO_STACK_LOCATION IrpSp; 2008 2009 stripe = ExAllocatePoolWithTag(NonPagedPool, sizeof(write_data_stripe), ALLOC_TAG); 2010 if (!stripe) { 2011 ERR("out of memory\n"); 2012 Status = STATUS_INSUFFICIENT_RESOURCES; 2013 goto end; 2014 } 2015 2016 if (stripes[i].start == stripes[i].end || !c->devices[i]->devobj) { 2017 stripe->status = WriteDataStatus_Ignore; 2018 stripe->Irp = NULL; 2019 stripe->buf = stripes[i].data; 2020 stripe->mdl = NULL; 2021 } else { 2022 stripe->context = (struct _write_data_context*)wtc; 2023 stripe->buf = stripes[i].data; 2024 stripe->device = c->devices[i]; 2025 RtlZeroMemory(&stripe->iosb, sizeof(IO_STATUS_BLOCK)); 2026 stripe->status = WriteDataStatus_Pending; 2027 stripe->mdl = stripes[i].mdl; 2028 2029 if (!Irp) { 2030 stripe->Irp = IoAllocateIrp(stripe->device->devobj->StackSize, false); 2031 2032 if (!stripe->Irp) { 2033 ERR("IoAllocateIrp failed\n"); 2034 ExFreePool(stripe); 2035 Status = STATUS_INSUFFICIENT_RESOURCES; 2036 goto end; 2037 } 2038 } else { 2039 stripe->Irp = IoMakeAssociatedIrp(Irp, stripe->device->devobj->StackSize); 2040 2041 if (!stripe->Irp) { 2042 ERR("IoMakeAssociatedIrp failed\n"); 2043 ExFreePool(stripe); 2044 Status = STATUS_INSUFFICIENT_RESOURCES; 2045 goto end; 2046 } 2047 } 2048 2049 IrpSp = IoGetNextIrpStackLocation(stripe->Irp); 2050 IrpSp->MajorFunction = IRP_MJ_WRITE; 2051 IrpSp->FileObject = stripe->device->fileobj; 2052 2053 if (stripe->device->devobj->Flags & DO_BUFFERED_IO) { 2054 stripe->Irp->AssociatedIrp.SystemBuffer = MmGetSystemAddressForMdlSafe(stripes[i].mdl, priority); 2055 2056 stripe->Irp->Flags = IRP_BUFFERED_IO; 2057 } else if (stripe->device->devobj->Flags & DO_DIRECT_IO) 2058 stripe->Irp->MdlAddress = stripe->mdl; 2059 else 2060 stripe->Irp->UserBuffer = MmGetSystemAddressForMdlSafe(stripes[i].mdl, priority); 2061 2062 #ifdef DEBUG_PARANOID 2063 if (stripes[i].end < stripes[i].start) { 2064 ERR("trying to write stripe with negative length (%I64x < %I64x)\n", stripes[i].end, stripes[i].start); 2065 int3; 2066 } 2067 #endif 2068 2069 IrpSp->Parameters.Write.Length = (ULONG)(stripes[i].end - stripes[i].start); 2070 IrpSp->Parameters.Write.ByteOffset.QuadPart = stripes[i].start + cis[i].offset; 2071 2072 total_writing += IrpSp->Parameters.Write.Length; 2073 2074 stripe->Irp->UserIosb = &stripe->iosb; 2075 wtc->stripes_left++; 2076 2077 IoSetCompletionRoutine(stripe->Irp, write_data_completion, stripe, true, true, true); 2078 } 2079 2080 InsertTailList(&wtc->stripes, &stripe->list_entry); 2081 } 2082 2083 if (diskacc) 2084 fFsRtlUpdateDiskCounters(0, total_writing); 2085 2086 Status = STATUS_SUCCESS; 2087 2088 end: 2089 2090 if (stripes) ExFreePool(stripes); 2091 2092 if (!NT_SUCCESS(Status)) 2093 free_write_data_stripes(wtc); 2094 2095 return Status; 2096 2097 prepare_failed: 2098 for (i = 0; i < c->chunk_item->num_stripes; i++) { 2099 if (stripes[i].mdl && (i == 0 || stripes[i].mdl != stripes[i-1].mdl)) { 2100 if (stripes[i].mdl->MdlFlags & MDL_PAGES_LOCKED) 2101 MmUnlockPages(stripes[i].mdl); 2102 2103 IoFreeMdl(stripes[i].mdl); 2104 } 2105 } 2106 2107 if (wtc->parity1_mdl) { 2108 if (wtc->parity1_mdl->MdlFlags & MDL_PAGES_LOCKED) 2109 MmUnlockPages(wtc->parity1_mdl); 2110 2111 IoFreeMdl(wtc->parity1_mdl); 2112 wtc->parity1_mdl = NULL; 2113 } 2114 2115 if (wtc->parity2_mdl) { 2116 if (wtc->parity2_mdl->MdlFlags & MDL_PAGES_LOCKED) 2117 MmUnlockPages(wtc->parity2_mdl); 2118 2119 IoFreeMdl(wtc->parity2_mdl); 2120 wtc->parity2_mdl = NULL; 2121 } 2122 2123 if (wtc->mdl) { 2124 if (wtc->mdl->MdlFlags & MDL_PAGES_LOCKED) 2125 MmUnlockPages(wtc->mdl); 2126 2127 IoFreeMdl(wtc->mdl); 2128 wtc->mdl = NULL; 2129 } 2130 2131 if (wtc->parity1) { 2132 ExFreePool(wtc->parity1); 2133 wtc->parity1 = NULL; 2134 } 2135 2136 if (wtc->parity2) { 2137 ExFreePool(wtc->parity2); 2138 wtc->parity2 = NULL; 2139 } 2140 2141 if (wtc->scratch) { 2142 ExFreePool(wtc->scratch); 2143 wtc->scratch = NULL; 2144 } 2145 2146 ExFreePool(stripes); 2147 return Status; 2148 } 2149 2150 void get_raid56_lock_range(chunk* c, uint64_t address, uint64_t length, uint64_t* lockaddr, uint64_t* locklen) { 2151 uint64_t startoff, endoff; 2152 uint16_t startoffstripe, endoffstripe, datastripes; 2153 2154 datastripes = c->chunk_item->num_stripes - (c->chunk_item->type & BLOCK_FLAG_RAID5 ? 1 : 2); 2155 2156 get_raid0_offset(address - c->offset, c->chunk_item->stripe_length, datastripes, &startoff, &startoffstripe); 2157 get_raid0_offset(address + length - c->offset - 1, c->chunk_item->stripe_length, datastripes, &endoff, &endoffstripe); 2158 2159 startoff -= startoff % c->chunk_item->stripe_length; 2160 endoff = sector_align(endoff, c->chunk_item->stripe_length); 2161 2162 *lockaddr = c->offset + (startoff * datastripes); 2163 *locklen = (endoff - startoff) * datastripes; 2164 } 2165 2166 NTSTATUS write_data_complete(device_extension* Vcb, uint64_t address, void* data, uint32_t length, PIRP Irp, chunk* c, bool file_write, uint64_t irp_offset, ULONG priority) { 2167 write_data_context wtc; 2168 NTSTATUS Status; 2169 uint64_t lockaddr, locklen; 2170 2171 KeInitializeEvent(&wtc.Event, NotificationEvent, false); 2172 InitializeListHead(&wtc.stripes); 2173 wtc.stripes_left = 0; 2174 wtc.parity1 = wtc.parity2 = wtc.scratch = NULL; 2175 wtc.mdl = wtc.parity1_mdl = wtc.parity2_mdl = NULL; 2176 2177 if (!c) { 2178 c = get_chunk_from_address(Vcb, address); 2179 if (!c) { 2180 ERR("could not get chunk for address %I64x\n", address); 2181 return STATUS_INTERNAL_ERROR; 2182 } 2183 } 2184 2185 if (c->chunk_item->type & BLOCK_FLAG_RAID5 || c->chunk_item->type & BLOCK_FLAG_RAID6) { 2186 get_raid56_lock_range(c, address, length, &lockaddr, &locklen); 2187 chunk_lock_range(Vcb, c, lockaddr, locklen); 2188 } 2189 2190 _SEH2_TRY { 2191 Status = write_data(Vcb, address, data, length, &wtc, Irp, c, file_write, irp_offset, priority); 2192 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 2193 Status = _SEH2_GetExceptionCode(); 2194 } _SEH2_END; 2195 2196 if (!NT_SUCCESS(Status)) { 2197 ERR("write_data returned %08x\n", Status); 2198 2199 if (c->chunk_item->type & BLOCK_FLAG_RAID5 || c->chunk_item->type & BLOCK_FLAG_RAID6) 2200 chunk_unlock_range(Vcb, c, lockaddr, locklen); 2201 2202 free_write_data_stripes(&wtc); 2203 return Status; 2204 } 2205 2206 if (wtc.stripes.Flink != &wtc.stripes) { 2207 // launch writes and wait 2208 LIST_ENTRY* le = wtc.stripes.Flink; 2209 bool no_wait = true; 2210 2211 while (le != &wtc.stripes) { 2212 write_data_stripe* stripe = CONTAINING_RECORD(le, write_data_stripe, list_entry); 2213 2214 if (stripe->status != WriteDataStatus_Ignore) { 2215 IoCallDriver(stripe->device->devobj, stripe->Irp); 2216 no_wait = false; 2217 } 2218 2219 le = le->Flink; 2220 } 2221 2222 if (!no_wait) 2223 KeWaitForSingleObject(&wtc.Event, Executive, KernelMode, false, NULL); 2224 2225 le = wtc.stripes.Flink; 2226 while (le != &wtc.stripes) { 2227 write_data_stripe* stripe = CONTAINING_RECORD(le, write_data_stripe, list_entry); 2228 2229 if (stripe->status != WriteDataStatus_Ignore && !NT_SUCCESS(stripe->iosb.Status)) { 2230 Status = stripe->iosb.Status; 2231 2232 log_device_error(Vcb, stripe->device, BTRFS_DEV_STAT_WRITE_ERRORS); 2233 break; 2234 } 2235 2236 le = le->Flink; 2237 } 2238 2239 free_write_data_stripes(&wtc); 2240 } 2241 2242 if (c->chunk_item->type & BLOCK_FLAG_RAID5 || c->chunk_item->type & BLOCK_FLAG_RAID6) 2243 chunk_unlock_range(Vcb, c, lockaddr, locklen); 2244 2245 return Status; 2246 } 2247 2248 _Function_class_(IO_COMPLETION_ROUTINE) 2249 static NTSTATUS __stdcall write_data_completion(PDEVICE_OBJECT DeviceObject, PIRP Irp, PVOID conptr) { 2250 write_data_stripe* stripe = conptr; 2251 write_data_context* context = (write_data_context*)stripe->context; 2252 LIST_ENTRY* le; 2253 2254 UNUSED(DeviceObject); 2255 2256 // FIXME - we need a lock here 2257 2258 if (stripe->status == WriteDataStatus_Cancelling) { 2259 stripe->status = WriteDataStatus_Cancelled; 2260 goto end; 2261 } 2262 2263 stripe->iosb = Irp->IoStatus; 2264 2265 if (NT_SUCCESS(Irp->IoStatus.Status)) { 2266 stripe->status = WriteDataStatus_Success; 2267 } else { 2268 le = context->stripes.Flink; 2269 2270 stripe->status = WriteDataStatus_Error; 2271 2272 while (le != &context->stripes) { 2273 write_data_stripe* s2 = CONTAINING_RECORD(le, write_data_stripe, list_entry); 2274 2275 if (s2->status == WriteDataStatus_Pending) { 2276 s2->status = WriteDataStatus_Cancelling; 2277 IoCancelIrp(s2->Irp); 2278 } 2279 2280 le = le->Flink; 2281 } 2282 } 2283 2284 end: 2285 if (InterlockedDecrement(&context->stripes_left) == 0) 2286 KeSetEvent(&context->Event, 0, false); 2287 2288 return STATUS_MORE_PROCESSING_REQUIRED; 2289 } 2290 2291 void free_write_data_stripes(write_data_context* wtc) { 2292 LIST_ENTRY* le; 2293 PMDL last_mdl = NULL; 2294 2295 if (wtc->parity1_mdl) { 2296 if (wtc->parity1_mdl->MdlFlags & MDL_PAGES_LOCKED) 2297 MmUnlockPages(wtc->parity1_mdl); 2298 2299 IoFreeMdl(wtc->parity1_mdl); 2300 } 2301 2302 if (wtc->parity2_mdl) { 2303 if (wtc->parity2_mdl->MdlFlags & MDL_PAGES_LOCKED) 2304 MmUnlockPages(wtc->parity2_mdl); 2305 2306 IoFreeMdl(wtc->parity2_mdl); 2307 } 2308 2309 if (wtc->mdl) { 2310 if (wtc->mdl->MdlFlags & MDL_PAGES_LOCKED) 2311 MmUnlockPages(wtc->mdl); 2312 2313 IoFreeMdl(wtc->mdl); 2314 } 2315 2316 if (wtc->parity1) 2317 ExFreePool(wtc->parity1); 2318 2319 if (wtc->parity2) 2320 ExFreePool(wtc->parity2); 2321 2322 if (wtc->scratch) 2323 ExFreePool(wtc->scratch); 2324 2325 le = wtc->stripes.Flink; 2326 while (le != &wtc->stripes) { 2327 write_data_stripe* stripe = CONTAINING_RECORD(le, write_data_stripe, list_entry); 2328 2329 if (stripe->mdl && stripe->mdl != last_mdl) { 2330 if (stripe->mdl->MdlFlags & MDL_PAGES_LOCKED) 2331 MmUnlockPages(stripe->mdl); 2332 2333 IoFreeMdl(stripe->mdl); 2334 } 2335 2336 last_mdl = stripe->mdl; 2337 2338 if (stripe->Irp) 2339 IoFreeIrp(stripe->Irp); 2340 2341 le = le->Flink; 2342 } 2343 2344 while (!IsListEmpty(&wtc->stripes)) { 2345 write_data_stripe* stripe = CONTAINING_RECORD(RemoveHeadList(&wtc->stripes), write_data_stripe, list_entry); 2346 2347 ExFreePool(stripe); 2348 } 2349 } 2350 2351 void add_extent(_In_ fcb* fcb, _In_ LIST_ENTRY* prevextle, _In_ __drv_aliasesMem extent* newext) { 2352 LIST_ENTRY* le = prevextle->Flink; 2353 2354 while (le != &fcb->extents) { 2355 extent* ext = CONTAINING_RECORD(le, extent, list_entry); 2356 2357 if (ext->offset >= newext->offset) { 2358 InsertHeadList(ext->list_entry.Blink, &newext->list_entry); 2359 return; 2360 } 2361 2362 le = le->Flink; 2363 } 2364 2365 InsertTailList(&fcb->extents, &newext->list_entry); 2366 } 2367 2368 NTSTATUS excise_extents(device_extension* Vcb, fcb* fcb, uint64_t start_data, uint64_t end_data, PIRP Irp, LIST_ENTRY* rollback) { 2369 NTSTATUS Status; 2370 LIST_ENTRY* le; 2371 2372 le = fcb->extents.Flink; 2373 2374 while (le != &fcb->extents) { 2375 LIST_ENTRY* le2 = le->Flink; 2376 extent* ext = CONTAINING_RECORD(le, extent, list_entry); 2377 EXTENT_DATA* ed = &ext->extent_data; 2378 EXTENT_DATA2* ed2 = NULL; 2379 uint64_t len; 2380 2381 if (!ext->ignore) { 2382 if (ed->type != EXTENT_TYPE_INLINE) 2383 ed2 = (EXTENT_DATA2*)ed->data; 2384 2385 len = ed->type == EXTENT_TYPE_INLINE ? ed->decoded_size : ed2->num_bytes; 2386 2387 if (ext->offset < end_data && ext->offset + len > start_data) { 2388 if (ed->type == EXTENT_TYPE_INLINE) { 2389 if (start_data <= ext->offset && end_data >= ext->offset + len) { // remove all 2390 remove_fcb_extent(fcb, ext, rollback); 2391 2392 fcb->inode_item.st_blocks -= len; 2393 fcb->inode_item_changed = true; 2394 } else { 2395 ERR("trying to split inline extent\n"); 2396 #ifdef DEBUG_PARANOID 2397 int3; 2398 #endif 2399 return STATUS_INTERNAL_ERROR; 2400 } 2401 } else if (ed->type != EXTENT_TYPE_INLINE) { 2402 if (start_data <= ext->offset && end_data >= ext->offset + len) { // remove all 2403 if (ed2->size != 0) { 2404 chunk* c; 2405 2406 fcb->inode_item.st_blocks -= len; 2407 fcb->inode_item_changed = true; 2408 2409 c = get_chunk_from_address(Vcb, ed2->address); 2410 2411 if (!c) { 2412 ERR("get_chunk_from_address(%I64x) failed\n", ed2->address); 2413 } else { 2414 Status = update_changed_extent_ref(Vcb, c, ed2->address, ed2->size, fcb->subvol->id, fcb->inode, ext->offset - ed2->offset, -1, 2415 fcb->inode_item.flags & BTRFS_INODE_NODATASUM, false, Irp); 2416 if (!NT_SUCCESS(Status)) { 2417 ERR("update_changed_extent_ref returned %08x\n", Status); 2418 goto end; 2419 } 2420 } 2421 } 2422 2423 remove_fcb_extent(fcb, ext, rollback); 2424 } else if (start_data <= ext->offset && end_data < ext->offset + len) { // remove beginning 2425 EXTENT_DATA2* ned2; 2426 extent* newext; 2427 2428 if (ed2->size != 0) { 2429 fcb->inode_item.st_blocks -= end_data - ext->offset; 2430 fcb->inode_item_changed = true; 2431 } 2432 2433 newext = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2), ALLOC_TAG); 2434 if (!newext) { 2435 ERR("out of memory\n"); 2436 Status = STATUS_INSUFFICIENT_RESOURCES; 2437 goto end; 2438 } 2439 2440 ned2 = (EXTENT_DATA2*)newext->extent_data.data; 2441 2442 newext->extent_data.generation = Vcb->superblock.generation; 2443 newext->extent_data.decoded_size = ed->decoded_size; 2444 newext->extent_data.compression = ed->compression; 2445 newext->extent_data.encryption = ed->encryption; 2446 newext->extent_data.encoding = ed->encoding; 2447 newext->extent_data.type = ed->type; 2448 ned2->address = ed2->address; 2449 ned2->size = ed2->size; 2450 ned2->offset = ed2->offset + (end_data - ext->offset); 2451 ned2->num_bytes = ed2->num_bytes - (end_data - ext->offset); 2452 2453 newext->offset = end_data; 2454 newext->datalen = sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2); 2455 newext->unique = ext->unique; 2456 newext->ignore = false; 2457 newext->inserted = true; 2458 2459 if (ext->csum) { 2460 if (ed->compression == BTRFS_COMPRESSION_NONE) { 2461 newext->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ned2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2462 if (!newext->csum) { 2463 ERR("out of memory\n"); 2464 Status = STATUS_INSUFFICIENT_RESOURCES; 2465 ExFreePool(newext); 2466 goto end; 2467 } 2468 2469 RtlCopyMemory(newext->csum, &ext->csum[(end_data - ext->offset) / Vcb->superblock.sector_size], 2470 (ULONG)(ned2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2471 } else { 2472 newext->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2473 if (!newext->csum) { 2474 ERR("out of memory\n"); 2475 Status = STATUS_INSUFFICIENT_RESOURCES; 2476 ExFreePool(newext); 2477 goto end; 2478 } 2479 2480 RtlCopyMemory(newext->csum, ext->csum, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2481 } 2482 } else 2483 newext->csum = NULL; 2484 2485 add_extent(fcb, &ext->list_entry, newext); 2486 2487 remove_fcb_extent(fcb, ext, rollback); 2488 } else if (start_data > ext->offset && end_data >= ext->offset + len) { // remove end 2489 EXTENT_DATA2* ned2; 2490 extent* newext; 2491 2492 if (ed2->size != 0) { 2493 fcb->inode_item.st_blocks -= ext->offset + len - start_data; 2494 fcb->inode_item_changed = true; 2495 } 2496 2497 newext = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2), ALLOC_TAG); 2498 if (!newext) { 2499 ERR("out of memory\n"); 2500 Status = STATUS_INSUFFICIENT_RESOURCES; 2501 goto end; 2502 } 2503 2504 ned2 = (EXTENT_DATA2*)newext->extent_data.data; 2505 2506 newext->extent_data.generation = Vcb->superblock.generation; 2507 newext->extent_data.decoded_size = ed->decoded_size; 2508 newext->extent_data.compression = ed->compression; 2509 newext->extent_data.encryption = ed->encryption; 2510 newext->extent_data.encoding = ed->encoding; 2511 newext->extent_data.type = ed->type; 2512 ned2->address = ed2->address; 2513 ned2->size = ed2->size; 2514 ned2->offset = ed2->offset; 2515 ned2->num_bytes = start_data - ext->offset; 2516 2517 newext->offset = ext->offset; 2518 newext->datalen = sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2); 2519 newext->unique = ext->unique; 2520 newext->ignore = false; 2521 newext->inserted = true; 2522 2523 if (ext->csum) { 2524 if (ed->compression == BTRFS_COMPRESSION_NONE) { 2525 newext->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ned2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2526 if (!newext->csum) { 2527 ERR("out of memory\n"); 2528 Status = STATUS_INSUFFICIENT_RESOURCES; 2529 ExFreePool(newext); 2530 goto end; 2531 } 2532 2533 RtlCopyMemory(newext->csum, ext->csum, (ULONG)(ned2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2534 } else { 2535 newext->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2536 if (!newext->csum) { 2537 ERR("out of memory\n"); 2538 Status = STATUS_INSUFFICIENT_RESOURCES; 2539 ExFreePool(newext); 2540 goto end; 2541 } 2542 2543 RtlCopyMemory(newext->csum, ext->csum, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2544 } 2545 } else 2546 newext->csum = NULL; 2547 2548 InsertHeadList(&ext->list_entry, &newext->list_entry); 2549 2550 remove_fcb_extent(fcb, ext, rollback); 2551 } else if (start_data > ext->offset && end_data < ext->offset + len) { // remove middle 2552 EXTENT_DATA2 *neda2, *nedb2; 2553 extent *newext1, *newext2; 2554 2555 if (ed2->size != 0) { 2556 chunk* c; 2557 2558 fcb->inode_item.st_blocks -= end_data - start_data; 2559 fcb->inode_item_changed = true; 2560 2561 c = get_chunk_from_address(Vcb, ed2->address); 2562 2563 if (!c) { 2564 ERR("get_chunk_from_address(%I64x) failed\n", ed2->address); 2565 } else { 2566 Status = update_changed_extent_ref(Vcb, c, ed2->address, ed2->size, fcb->subvol->id, fcb->inode, ext->offset - ed2->offset, 1, 2567 fcb->inode_item.flags & BTRFS_INODE_NODATASUM, false, Irp); 2568 if (!NT_SUCCESS(Status)) { 2569 ERR("update_changed_extent_ref returned %08x\n", Status); 2570 goto end; 2571 } 2572 } 2573 } 2574 2575 newext1 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2), ALLOC_TAG); 2576 if (!newext1) { 2577 ERR("out of memory\n"); 2578 Status = STATUS_INSUFFICIENT_RESOURCES; 2579 goto end; 2580 } 2581 2582 newext2 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2), ALLOC_TAG); 2583 if (!newext2) { 2584 ERR("out of memory\n"); 2585 Status = STATUS_INSUFFICIENT_RESOURCES; 2586 ExFreePool(newext1); 2587 goto end; 2588 } 2589 2590 neda2 = (EXTENT_DATA2*)newext1->extent_data.data; 2591 2592 newext1->extent_data.generation = Vcb->superblock.generation; 2593 newext1->extent_data.decoded_size = ed->decoded_size; 2594 newext1->extent_data.compression = ed->compression; 2595 newext1->extent_data.encryption = ed->encryption; 2596 newext1->extent_data.encoding = ed->encoding; 2597 newext1->extent_data.type = ed->type; 2598 neda2->address = ed2->address; 2599 neda2->size = ed2->size; 2600 neda2->offset = ed2->offset; 2601 neda2->num_bytes = start_data - ext->offset; 2602 2603 nedb2 = (EXTENT_DATA2*)newext2->extent_data.data; 2604 2605 newext2->extent_data.generation = Vcb->superblock.generation; 2606 newext2->extent_data.decoded_size = ed->decoded_size; 2607 newext2->extent_data.compression = ed->compression; 2608 newext2->extent_data.encryption = ed->encryption; 2609 newext2->extent_data.encoding = ed->encoding; 2610 newext2->extent_data.type = ed->type; 2611 nedb2->address = ed2->address; 2612 nedb2->size = ed2->size; 2613 nedb2->offset = ed2->offset + (end_data - ext->offset); 2614 nedb2->num_bytes = ext->offset + len - end_data; 2615 2616 newext1->offset = ext->offset; 2617 newext1->datalen = sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2); 2618 newext1->unique = ext->unique; 2619 newext1->ignore = false; 2620 newext1->inserted = true; 2621 2622 newext2->offset = end_data; 2623 newext2->datalen = sizeof(EXTENT_DATA) - 1 + sizeof(EXTENT_DATA2); 2624 newext2->unique = ext->unique; 2625 newext2->ignore = false; 2626 newext2->inserted = true; 2627 2628 if (ext->csum) { 2629 if (ed->compression == BTRFS_COMPRESSION_NONE) { 2630 newext1->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(neda2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2631 if (!newext1->csum) { 2632 ERR("out of memory\n"); 2633 Status = STATUS_INSUFFICIENT_RESOURCES; 2634 ExFreePool(newext1); 2635 ExFreePool(newext2); 2636 goto end; 2637 } 2638 2639 newext2->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(nedb2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2640 if (!newext2->csum) { 2641 ERR("out of memory\n"); 2642 Status = STATUS_INSUFFICIENT_RESOURCES; 2643 ExFreePool(newext1->csum); 2644 ExFreePool(newext1); 2645 ExFreePool(newext2); 2646 goto end; 2647 } 2648 2649 RtlCopyMemory(newext1->csum, ext->csum, (ULONG)(neda2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2650 RtlCopyMemory(newext2->csum, &ext->csum[(end_data - ext->offset) / Vcb->superblock.sector_size], 2651 (ULONG)(nedb2->num_bytes * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2652 } else { 2653 newext1->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2654 if (!newext1->csum) { 2655 ERR("out of memory\n"); 2656 Status = STATUS_INSUFFICIENT_RESOURCES; 2657 ExFreePool(newext1); 2658 ExFreePool(newext2); 2659 goto end; 2660 } 2661 2662 newext2->csum = ExAllocatePoolWithTag(PagedPool, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size), ALLOC_TAG); 2663 if (!newext2->csum) { 2664 ERR("out of memory\n"); 2665 Status = STATUS_INSUFFICIENT_RESOURCES; 2666 ExFreePool(newext1->csum); 2667 ExFreePool(newext1); 2668 ExFreePool(newext2); 2669 goto end; 2670 } 2671 2672 RtlCopyMemory(newext1->csum, ext->csum, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2673 RtlCopyMemory(newext2->csum, ext->csum, (ULONG)(ed2->size * sizeof(uint32_t) / Vcb->superblock.sector_size)); 2674 } 2675 } else { 2676 newext1->csum = NULL; 2677 newext2->csum = NULL; 2678 } 2679 2680 InsertHeadList(&ext->list_entry, &newext1->list_entry); 2681 add_extent(fcb, &newext1->list_entry, newext2); 2682 2683 remove_fcb_extent(fcb, ext, rollback); 2684 } 2685 } 2686 } 2687 } 2688 2689 le = le2; 2690 } 2691 2692 Status = STATUS_SUCCESS; 2693 2694 end: 2695 fcb->extents_changed = true; 2696 mark_fcb_dirty(fcb); 2697 2698 return Status; 2699 } 2700 2701 void add_insert_extent_rollback(LIST_ENTRY* rollback, fcb* fcb, extent* ext) { 2702 rollback_extent* re; 2703 2704 re = ExAllocatePoolWithTag(NonPagedPool, sizeof(rollback_extent), ALLOC_TAG); 2705 if (!re) { 2706 ERR("out of memory\n"); 2707 return; 2708 } 2709 2710 re->fcb = fcb; 2711 re->ext = ext; 2712 2713 add_rollback(rollback, ROLLBACK_INSERT_EXTENT, re); 2714 } 2715 2716 #ifdef _MSC_VER 2717 #pragma warning(push) 2718 #pragma warning(suppress: 28194) 2719 #endif 2720 NTSTATUS add_extent_to_fcb(_In_ fcb* fcb, _In_ uint64_t offset, _In_reads_bytes_(edsize) EXTENT_DATA* ed, _In_ uint16_t edsize, 2721 _In_ bool unique, _In_opt_ _When_(return >= 0, __drv_aliasesMem) uint32_t* csum, _In_ LIST_ENTRY* rollback) { 2722 extent* ext; 2723 LIST_ENTRY* le; 2724 2725 ext = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + edsize, ALLOC_TAG); 2726 if (!ext) { 2727 ERR("out of memory\n"); 2728 return STATUS_INSUFFICIENT_RESOURCES; 2729 } 2730 2731 ext->offset = offset; 2732 ext->datalen = edsize; 2733 ext->unique = unique; 2734 ext->ignore = false; 2735 ext->inserted = true; 2736 ext->csum = csum; 2737 2738 RtlCopyMemory(&ext->extent_data, ed, edsize); 2739 2740 le = fcb->extents.Flink; 2741 while (le != &fcb->extents) { 2742 extent* oldext = CONTAINING_RECORD(le, extent, list_entry); 2743 2744 if (oldext->offset >= offset) { 2745 InsertHeadList(le->Blink, &ext->list_entry); 2746 goto end; 2747 } 2748 2749 le = le->Flink; 2750 } 2751 2752 InsertTailList(&fcb->extents, &ext->list_entry); 2753 2754 end: 2755 add_insert_extent_rollback(rollback, fcb, ext); 2756 2757 return STATUS_SUCCESS; 2758 } 2759 #ifdef _MSC_VER 2760 #pragma warning(pop) 2761 #endif 2762 2763 static void remove_fcb_extent(fcb* fcb, extent* ext, LIST_ENTRY* rollback) { 2764 if (!ext->ignore) { 2765 rollback_extent* re; 2766 2767 ext->ignore = true; 2768 2769 re = ExAllocatePoolWithTag(NonPagedPool, sizeof(rollback_extent), ALLOC_TAG); 2770 if (!re) { 2771 ERR("out of memory\n"); 2772 return; 2773 } 2774 2775 re->fcb = fcb; 2776 re->ext = ext; 2777 2778 add_rollback(rollback, ROLLBACK_DELETE_EXTENT, re); 2779 } 2780 } 2781 2782 NTSTATUS calc_csum(_In_ device_extension* Vcb, _In_reads_bytes_(sectors*Vcb->superblock.sector_size) uint8_t* data, 2783 _In_ uint32_t sectors, _Out_writes_bytes_(sectors*sizeof(uint32_t)) uint32_t* csum) { 2784 NTSTATUS Status; 2785 calc_job* cj; 2786 2787 // From experimenting, it seems that 40 sectors is roughly the crossover 2788 // point where offloading the crc32 calculation becomes worth it. 2789 2790 if (sectors < 40 || get_num_of_processors() < 2) { 2791 ULONG j; 2792 2793 for (j = 0; j < sectors; j++) { 2794 csum[j] = ~calc_crc32c(0xffffffff, data + (j * Vcb->superblock.sector_size), Vcb->superblock.sector_size); 2795 } 2796 2797 return STATUS_SUCCESS; 2798 } 2799 2800 Status = add_calc_job(Vcb, data, sectors, csum, &cj); 2801 if (!NT_SUCCESS(Status)) { 2802 ERR("add_calc_job returned %08x\n", Status); 2803 return Status; 2804 } 2805 2806 KeWaitForSingleObject(&cj->event, Executive, KernelMode, false, NULL); 2807 free_calc_job(cj); 2808 2809 return STATUS_SUCCESS; 2810 } 2811 2812 _Requires_lock_held_(c->lock) 2813 _When_(return != 0, _Releases_lock_(c->lock)) 2814 bool insert_extent_chunk(_In_ device_extension* Vcb, _In_ fcb* fcb, _In_ chunk* c, _In_ uint64_t start_data, _In_ uint64_t length, _In_ bool prealloc, _In_opt_ void* data, 2815 _In_opt_ PIRP Irp, _In_ LIST_ENTRY* rollback, _In_ uint8_t compression, _In_ uint64_t decoded_size, _In_ bool file_write, _In_ uint64_t irp_offset) { 2816 uint64_t address; 2817 NTSTATUS Status; 2818 EXTENT_DATA* ed; 2819 EXTENT_DATA2* ed2; 2820 uint16_t edsize = (uint16_t)(offsetof(EXTENT_DATA, data[0]) + sizeof(EXTENT_DATA2)); 2821 uint32_t* csum = NULL; 2822 2823 TRACE("(%p, (%I64x, %I64x), %I64x, %I64x, %I64x, %u, %p, %p)\n", Vcb, fcb->subvol->id, fcb->inode, c->offset, start_data, length, prealloc, data, rollback); 2824 2825 if (!find_data_address_in_chunk(Vcb, c, length, &address)) 2826 return false; 2827 2828 // add extent data to inode 2829 ed = ExAllocatePoolWithTag(PagedPool, edsize, ALLOC_TAG); 2830 if (!ed) { 2831 ERR("out of memory\n"); 2832 return false; 2833 } 2834 2835 ed->generation = Vcb->superblock.generation; 2836 ed->decoded_size = decoded_size; 2837 ed->compression = compression; 2838 ed->encryption = BTRFS_ENCRYPTION_NONE; 2839 ed->encoding = BTRFS_ENCODING_NONE; 2840 ed->type = prealloc ? EXTENT_TYPE_PREALLOC : EXTENT_TYPE_REGULAR; 2841 2842 ed2 = (EXTENT_DATA2*)ed->data; 2843 ed2->address = address; 2844 ed2->size = length; 2845 ed2->offset = 0; 2846 ed2->num_bytes = decoded_size; 2847 2848 if (!prealloc && data && !(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 2849 ULONG sl = (ULONG)(length / Vcb->superblock.sector_size); 2850 2851 csum = ExAllocatePoolWithTag(PagedPool, sl * sizeof(uint32_t), ALLOC_TAG); 2852 if (!csum) { 2853 ERR("out of memory\n"); 2854 ExFreePool(ed); 2855 return false; 2856 } 2857 2858 Status = calc_csum(Vcb, data, sl, csum); 2859 if (!NT_SUCCESS(Status)) { 2860 ERR("calc_csum returned %08x\n", Status); 2861 ExFreePool(csum); 2862 ExFreePool(ed); 2863 return false; 2864 } 2865 } 2866 2867 Status = add_extent_to_fcb(fcb, start_data, ed, edsize, true, csum, rollback); 2868 if (!NT_SUCCESS(Status)) { 2869 ERR("add_extent_to_fcb returned %08x\n", Status); 2870 if (csum) ExFreePool(csum); 2871 ExFreePool(ed); 2872 return false; 2873 } 2874 2875 ExFreePool(ed); 2876 2877 c->used += length; 2878 space_list_subtract(c, false, address, length, rollback); 2879 2880 fcb->inode_item.st_blocks += decoded_size; 2881 2882 fcb->extents_changed = true; 2883 fcb->inode_item_changed = true; 2884 mark_fcb_dirty(fcb); 2885 2886 ExAcquireResourceExclusiveLite(&c->changed_extents_lock, true); 2887 2888 add_changed_extent_ref(c, address, length, fcb->subvol->id, fcb->inode, start_data, 1, fcb->inode_item.flags & BTRFS_INODE_NODATASUM); 2889 2890 ExReleaseResourceLite(&c->changed_extents_lock); 2891 2892 release_chunk_lock(c, Vcb); 2893 2894 if (data) { 2895 Status = write_data_complete(Vcb, address, data, (uint32_t)length, Irp, NULL, file_write, irp_offset, 2896 fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE ? HighPagePriority : NormalPagePriority); 2897 if (!NT_SUCCESS(Status)) 2898 ERR("write_data_complete returned %08x\n", Status); 2899 } 2900 2901 return true; 2902 } 2903 2904 static bool try_extend_data(device_extension* Vcb, fcb* fcb, uint64_t start_data, uint64_t length, void* data, 2905 PIRP Irp, uint64_t* written, bool file_write, uint64_t irp_offset, LIST_ENTRY* rollback) { 2906 bool success = false; 2907 EXTENT_DATA* ed; 2908 EXTENT_DATA2* ed2; 2909 chunk* c; 2910 LIST_ENTRY* le; 2911 extent* ext = NULL; 2912 2913 le = fcb->extents.Flink; 2914 2915 while (le != &fcb->extents) { 2916 extent* nextext = CONTAINING_RECORD(le, extent, list_entry); 2917 2918 if (!nextext->ignore) { 2919 if (nextext->offset == start_data) { 2920 ext = nextext; 2921 break; 2922 } else if (nextext->offset > start_data) 2923 break; 2924 2925 ext = nextext; 2926 } 2927 2928 le = le->Flink; 2929 } 2930 2931 if (!ext) 2932 return false; 2933 2934 ed = &ext->extent_data; 2935 2936 if (ed->type != EXTENT_TYPE_REGULAR && ed->type != EXTENT_TYPE_PREALLOC) { 2937 TRACE("not extending extent which is not regular or prealloc\n"); 2938 return false; 2939 } 2940 2941 ed2 = (EXTENT_DATA2*)ed->data; 2942 2943 if (ext->offset + ed2->num_bytes != start_data) { 2944 TRACE("last EXTENT_DATA does not run up to start_data (%I64x + %I64x != %I64x)\n", ext->offset, ed2->num_bytes, start_data); 2945 return false; 2946 } 2947 2948 c = get_chunk_from_address(Vcb, ed2->address); 2949 2950 if (c->reloc || c->readonly || c->chunk_item->type != Vcb->data_flags) 2951 return false; 2952 2953 acquire_chunk_lock(c, Vcb); 2954 2955 if (length > c->chunk_item->size - c->used) { 2956 release_chunk_lock(c, Vcb); 2957 return false; 2958 } 2959 2960 if (!c->cache_loaded) { 2961 NTSTATUS Status = load_cache_chunk(Vcb, c, NULL); 2962 2963 if (!NT_SUCCESS(Status)) { 2964 ERR("load_cache_chunk returned %08x\n", Status); 2965 release_chunk_lock(c, Vcb); 2966 return false; 2967 } 2968 } 2969 2970 le = c->space.Flink; 2971 while (le != &c->space) { 2972 space* s = CONTAINING_RECORD(le, space, list_entry); 2973 2974 if (s->address == ed2->address + ed2->size) { 2975 uint64_t newlen = min(min(s->size, length), MAX_EXTENT_SIZE); 2976 2977 success = insert_extent_chunk(Vcb, fcb, c, start_data, newlen, false, data, Irp, rollback, BTRFS_COMPRESSION_NONE, newlen, file_write, irp_offset); 2978 2979 if (success) 2980 *written += newlen; 2981 else 2982 release_chunk_lock(c, Vcb); 2983 2984 return success; 2985 } else if (s->address > ed2->address + ed2->size) 2986 break; 2987 2988 le = le->Flink; 2989 } 2990 2991 release_chunk_lock(c, Vcb); 2992 2993 return false; 2994 } 2995 2996 static NTSTATUS insert_chunk_fragmented(fcb* fcb, uint64_t start, uint64_t length, uint8_t* data, bool prealloc, LIST_ENTRY* rollback) { 2997 LIST_ENTRY* le; 2998 uint64_t flags = fcb->Vcb->data_flags; 2999 bool page_file = fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE; 3000 NTSTATUS Status; 3001 chunk* c; 3002 3003 ExAcquireResourceSharedLite(&fcb->Vcb->chunk_lock, true); 3004 3005 // first create as many chunks as we can 3006 do { 3007 Status = alloc_chunk(fcb->Vcb, flags, &c, false); 3008 } while (NT_SUCCESS(Status)); 3009 3010 if (Status != STATUS_DISK_FULL) { 3011 ERR("alloc_chunk returned %08x\n", Status); 3012 ExReleaseResourceLite(&fcb->Vcb->chunk_lock); 3013 return Status; 3014 } 3015 3016 le = fcb->Vcb->chunks.Flink; 3017 while (le != &fcb->Vcb->chunks) { 3018 c = CONTAINING_RECORD(le, chunk, list_entry); 3019 3020 if (!c->readonly && !c->reloc) { 3021 acquire_chunk_lock(c, fcb->Vcb); 3022 3023 if (c->chunk_item->type == flags) { 3024 while (!IsListEmpty(&c->space_size) && length > 0) { 3025 space* s = CONTAINING_RECORD(c->space_size.Flink, space, list_entry_size); 3026 uint64_t extlen = min(length, s->size); 3027 3028 if (insert_extent_chunk(fcb->Vcb, fcb, c, start, extlen, prealloc && !page_file, data, NULL, rollback, BTRFS_COMPRESSION_NONE, extlen, false, 0)) { 3029 start += extlen; 3030 length -= extlen; 3031 if (data) data += extlen; 3032 3033 acquire_chunk_lock(c, fcb->Vcb); 3034 } 3035 } 3036 } 3037 3038 release_chunk_lock(c, fcb->Vcb); 3039 3040 if (length == 0) 3041 break; 3042 } 3043 3044 le = le->Flink; 3045 } 3046 3047 ExReleaseResourceLite(&fcb->Vcb->chunk_lock); 3048 3049 return length == 0 ? STATUS_SUCCESS : STATUS_DISK_FULL; 3050 } 3051 3052 static NTSTATUS insert_prealloc_extent(fcb* fcb, uint64_t start, uint64_t length, LIST_ENTRY* rollback) { 3053 LIST_ENTRY* le; 3054 chunk* c; 3055 uint64_t flags; 3056 NTSTATUS Status; 3057 bool page_file = fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE; 3058 3059 flags = fcb->Vcb->data_flags; 3060 3061 do { 3062 uint64_t extlen = min(MAX_EXTENT_SIZE, length); 3063 3064 ExAcquireResourceSharedLite(&fcb->Vcb->chunk_lock, true); 3065 3066 le = fcb->Vcb->chunks.Flink; 3067 while (le != &fcb->Vcb->chunks) { 3068 c = CONTAINING_RECORD(le, chunk, list_entry); 3069 3070 if (!c->readonly && !c->reloc) { 3071 acquire_chunk_lock(c, fcb->Vcb); 3072 3073 if (c->chunk_item->type == flags && (c->chunk_item->size - c->used) >= extlen) { 3074 if (insert_extent_chunk(fcb->Vcb, fcb, c, start, extlen, !page_file, NULL, NULL, rollback, BTRFS_COMPRESSION_NONE, extlen, false, 0)) { 3075 ExReleaseResourceLite(&fcb->Vcb->chunk_lock); 3076 goto cont; 3077 } 3078 } 3079 3080 release_chunk_lock(c, fcb->Vcb); 3081 } 3082 3083 le = le->Flink; 3084 } 3085 3086 ExReleaseResourceLite(&fcb->Vcb->chunk_lock); 3087 3088 ExAcquireResourceExclusiveLite(&fcb->Vcb->chunk_lock, true); 3089 3090 Status = alloc_chunk(fcb->Vcb, flags, &c, false); 3091 3092 ExReleaseResourceLite(&fcb->Vcb->chunk_lock); 3093 3094 if (!NT_SUCCESS(Status)) { 3095 ERR("alloc_chunk returned %08x\n", Status); 3096 goto end; 3097 } 3098 3099 acquire_chunk_lock(c, fcb->Vcb); 3100 3101 if (c->chunk_item->type == flags && (c->chunk_item->size - c->used) >= extlen) { 3102 if (insert_extent_chunk(fcb->Vcb, fcb, c, start, extlen, !page_file, NULL, NULL, rollback, BTRFS_COMPRESSION_NONE, extlen, false, 0)) 3103 goto cont; 3104 } 3105 3106 release_chunk_lock(c, fcb->Vcb); 3107 3108 Status = insert_chunk_fragmented(fcb, start, length, NULL, true, rollback); 3109 if (!NT_SUCCESS(Status)) 3110 ERR("insert_chunk_fragmented returned %08x\n", Status); 3111 3112 goto end; 3113 3114 cont: 3115 length -= extlen; 3116 start += extlen; 3117 } while (length > 0); 3118 3119 Status = STATUS_SUCCESS; 3120 3121 end: 3122 return Status; 3123 } 3124 3125 static NTSTATUS insert_extent(device_extension* Vcb, fcb* fcb, uint64_t start_data, uint64_t length, void* data, 3126 PIRP Irp, bool file_write, uint64_t irp_offset, LIST_ENTRY* rollback) { 3127 NTSTATUS Status; 3128 LIST_ENTRY* le; 3129 chunk* c; 3130 uint64_t flags, orig_length = length, written = 0; 3131 3132 TRACE("(%p, (%I64x, %I64x), %I64x, %I64x, %p)\n", Vcb, fcb->subvol->id, fcb->inode, start_data, length, data); 3133 3134 if (start_data > 0) { 3135 try_extend_data(Vcb, fcb, start_data, length, data, Irp, &written, file_write, irp_offset, rollback); 3136 3137 if (written == length) 3138 return STATUS_SUCCESS; 3139 else if (written > 0) { 3140 start_data += written; 3141 irp_offset += written; 3142 length -= written; 3143 data = &((uint8_t*)data)[written]; 3144 } 3145 } 3146 3147 flags = Vcb->data_flags; 3148 3149 while (written < orig_length) { 3150 uint64_t newlen = min(length, MAX_EXTENT_SIZE); 3151 bool done = false; 3152 3153 // Rather than necessarily writing the whole extent at once, we deal with it in blocks of 128 MB. 3154 // First, see if we can write the extent part to an existing chunk. 3155 3156 ExAcquireResourceSharedLite(&Vcb->chunk_lock, true); 3157 3158 le = Vcb->chunks.Flink; 3159 while (le != &Vcb->chunks) { 3160 c = CONTAINING_RECORD(le, chunk, list_entry); 3161 3162 if (!c->readonly && !c->reloc) { 3163 acquire_chunk_lock(c, Vcb); 3164 3165 if (c->chunk_item->type == flags && (c->chunk_item->size - c->used) >= newlen && 3166 insert_extent_chunk(Vcb, fcb, c, start_data, newlen, false, data, Irp, rollback, BTRFS_COMPRESSION_NONE, newlen, file_write, irp_offset)) { 3167 written += newlen; 3168 3169 if (written == orig_length) { 3170 ExReleaseResourceLite(&Vcb->chunk_lock); 3171 return STATUS_SUCCESS; 3172 } else { 3173 done = true; 3174 start_data += newlen; 3175 irp_offset += newlen; 3176 length -= newlen; 3177 data = &((uint8_t*)data)[newlen]; 3178 break; 3179 } 3180 } else 3181 release_chunk_lock(c, Vcb); 3182 } 3183 3184 le = le->Flink; 3185 } 3186 3187 ExReleaseResourceLite(&Vcb->chunk_lock); 3188 3189 if (done) continue; 3190 3191 // Otherwise, see if we can put it in a new chunk. 3192 3193 ExAcquireResourceExclusiveLite(&Vcb->chunk_lock, true); 3194 3195 Status = alloc_chunk(Vcb, flags, &c, false); 3196 3197 ExReleaseResourceLite(&Vcb->chunk_lock); 3198 3199 if (!NT_SUCCESS(Status)) { 3200 ERR("alloc_chunk returned %08x\n", Status); 3201 return Status; 3202 } 3203 3204 if (c) { 3205 acquire_chunk_lock(c, Vcb); 3206 3207 if (c->chunk_item->type == flags && (c->chunk_item->size - c->used) >= newlen && 3208 insert_extent_chunk(Vcb, fcb, c, start_data, newlen, false, data, Irp, rollback, BTRFS_COMPRESSION_NONE, newlen, file_write, irp_offset)) { 3209 written += newlen; 3210 3211 if (written == orig_length) 3212 return STATUS_SUCCESS; 3213 else { 3214 done = true; 3215 start_data += newlen; 3216 irp_offset += newlen; 3217 length -= newlen; 3218 data = &((uint8_t*)data)[newlen]; 3219 } 3220 } else 3221 release_chunk_lock(c, Vcb); 3222 } 3223 3224 if (!done) { 3225 Status = insert_chunk_fragmented(fcb, start_data, length, data, false, rollback); 3226 if (!NT_SUCCESS(Status)) 3227 ERR("insert_chunk_fragmented returned %08x\n", Status); 3228 3229 return Status; 3230 } 3231 } 3232 3233 return STATUS_DISK_FULL; 3234 } 3235 3236 NTSTATUS truncate_file(fcb* fcb, uint64_t end, PIRP Irp, LIST_ENTRY* rollback) { 3237 NTSTATUS Status; 3238 3239 // FIXME - convert into inline extent if short enough 3240 3241 if (end > 0 && fcb_is_inline(fcb)) { 3242 uint8_t* buf; 3243 bool make_inline = end <= fcb->Vcb->options.max_inline; 3244 3245 buf = ExAllocatePoolWithTag(PagedPool, (ULONG)(make_inline ? (offsetof(EXTENT_DATA, data[0]) + end) : sector_align(end, fcb->Vcb->superblock.sector_size)), ALLOC_TAG); 3246 if (!buf) { 3247 ERR("out of memory\n"); 3248 return STATUS_INSUFFICIENT_RESOURCES; 3249 } 3250 3251 Status = read_file(fcb, make_inline ? (buf + offsetof(EXTENT_DATA, data[0])) : buf, 0, end, NULL, Irp); 3252 if (!NT_SUCCESS(Status)) { 3253 ERR("read_file returned %08x\n", Status); 3254 ExFreePool(buf); 3255 return Status; 3256 } 3257 3258 Status = excise_extents(fcb->Vcb, fcb, 0, fcb->inode_item.st_size, Irp, rollback); 3259 if (!NT_SUCCESS(Status)) { 3260 ERR("excise_extents returned %08x\n", Status); 3261 ExFreePool(buf); 3262 return Status; 3263 } 3264 3265 if (!make_inline) { 3266 RtlZeroMemory(buf + end, (ULONG)(sector_align(end, fcb->Vcb->superblock.sector_size) - end)); 3267 3268 Status = do_write_file(fcb, 0, sector_align(end, fcb->Vcb->superblock.sector_size), buf, Irp, false, 0, rollback); 3269 if (!NT_SUCCESS(Status)) { 3270 ERR("do_write_file returned %08x\n", Status); 3271 ExFreePool(buf); 3272 return Status; 3273 } 3274 } else { 3275 EXTENT_DATA* ed = (EXTENT_DATA*)buf; 3276 3277 ed->generation = fcb->Vcb->superblock.generation; 3278 ed->decoded_size = end; 3279 ed->compression = BTRFS_COMPRESSION_NONE; 3280 ed->encryption = BTRFS_ENCRYPTION_NONE; 3281 ed->encoding = BTRFS_ENCODING_NONE; 3282 ed->type = EXTENT_TYPE_INLINE; 3283 3284 Status = add_extent_to_fcb(fcb, 0, ed, (uint16_t)(offsetof(EXTENT_DATA, data[0]) + end), false, NULL, rollback); 3285 if (!NT_SUCCESS(Status)) { 3286 ERR("add_extent_to_fcb returned %08x\n", Status); 3287 ExFreePool(buf); 3288 return Status; 3289 } 3290 3291 fcb->inode_item.st_blocks += end; 3292 } 3293 3294 ExFreePool(buf); 3295 return STATUS_SUCCESS; 3296 } 3297 3298 Status = excise_extents(fcb->Vcb, fcb, sector_align(end, fcb->Vcb->superblock.sector_size), 3299 sector_align(fcb->inode_item.st_size, fcb->Vcb->superblock.sector_size), Irp, rollback); 3300 if (!NT_SUCCESS(Status)) { 3301 ERR("excise_extents returned %08x\n", Status); 3302 return Status; 3303 } 3304 3305 fcb->inode_item.st_size = end; 3306 fcb->inode_item_changed = true; 3307 TRACE("setting st_size to %I64x\n", end); 3308 3309 fcb->Header.AllocationSize.QuadPart = sector_align(fcb->inode_item.st_size, fcb->Vcb->superblock.sector_size); 3310 fcb->Header.FileSize.QuadPart = fcb->inode_item.st_size; 3311 fcb->Header.ValidDataLength.QuadPart = fcb->inode_item.st_size; 3312 // FIXME - inform cache manager of this 3313 3314 TRACE("fcb %p FileSize = %I64x\n", fcb, fcb->Header.FileSize.QuadPart); 3315 3316 return STATUS_SUCCESS; 3317 } 3318 3319 NTSTATUS extend_file(fcb* fcb, file_ref* fileref, uint64_t end, bool prealloc, PIRP Irp, LIST_ENTRY* rollback) { 3320 uint64_t oldalloc, newalloc; 3321 bool cur_inline; 3322 NTSTATUS Status; 3323 3324 TRACE("(%p, %p, %x, %u)\n", fcb, fileref, end, prealloc); 3325 3326 if (fcb->ads) { 3327 if (end > 0xffff) 3328 return STATUS_DISK_FULL; 3329 3330 return stream_set_end_of_file_information(fcb->Vcb, (uint16_t)end, fcb, fileref, false); 3331 } else { 3332 extent* ext = NULL; 3333 LIST_ENTRY* le; 3334 3335 le = fcb->extents.Blink; 3336 while (le != &fcb->extents) { 3337 extent* ext2 = CONTAINING_RECORD(le, extent, list_entry); 3338 3339 if (!ext2->ignore) { 3340 ext = ext2; 3341 break; 3342 } 3343 3344 le = le->Blink; 3345 } 3346 3347 oldalloc = 0; 3348 if (ext) { 3349 EXTENT_DATA* ed = &ext->extent_data; 3350 EXTENT_DATA2* ed2 = (EXTENT_DATA2*)ed->data; 3351 3352 oldalloc = ext->offset + (ed->type == EXTENT_TYPE_INLINE ? ed->decoded_size : ed2->num_bytes); 3353 cur_inline = ed->type == EXTENT_TYPE_INLINE; 3354 3355 if (cur_inline && end > fcb->Vcb->options.max_inline) { 3356 uint64_t origlength, length; 3357 uint8_t* data; 3358 3359 TRACE("giving inline file proper extents\n"); 3360 3361 origlength = ed->decoded_size; 3362 3363 cur_inline = false; 3364 3365 length = sector_align(origlength, fcb->Vcb->superblock.sector_size); 3366 3367 data = ExAllocatePoolWithTag(PagedPool, (ULONG)length, ALLOC_TAG); 3368 if (!data) { 3369 ERR("could not allocate %I64x bytes for data\n", length); 3370 return STATUS_INSUFFICIENT_RESOURCES; 3371 } 3372 3373 Status = read_file(fcb, data, 0, origlength, NULL, Irp); 3374 if (!NT_SUCCESS(Status)) { 3375 ERR("read_file returned %08x\n", Status); 3376 ExFreePool(data); 3377 return Status; 3378 } 3379 3380 RtlZeroMemory(data + origlength, (ULONG)(length - origlength)); 3381 3382 Status = excise_extents(fcb->Vcb, fcb, 0, fcb->inode_item.st_size, Irp, rollback); 3383 if (!NT_SUCCESS(Status)) { 3384 ERR("excise_extents returned %08x\n", Status); 3385 ExFreePool(data); 3386 return Status; 3387 } 3388 3389 Status = do_write_file(fcb, 0, length, data, Irp, false, 0, rollback); 3390 if (!NT_SUCCESS(Status)) { 3391 ERR("do_write_file returned %08x\n", Status); 3392 ExFreePool(data); 3393 return Status; 3394 } 3395 3396 oldalloc = ext->offset + length; 3397 3398 ExFreePool(data); 3399 } 3400 3401 if (cur_inline) { 3402 uint16_t edsize; 3403 3404 if (end > oldalloc) { 3405 edsize = (uint16_t)(offsetof(EXTENT_DATA, data[0]) + end - ext->offset); 3406 ed = ExAllocatePoolWithTag(PagedPool, edsize, ALLOC_TAG); 3407 3408 if (!ed) { 3409 ERR("out of memory\n"); 3410 return STATUS_INSUFFICIENT_RESOURCES; 3411 } 3412 3413 ed->generation = fcb->Vcb->superblock.generation; 3414 ed->decoded_size = end - ext->offset; 3415 ed->compression = BTRFS_COMPRESSION_NONE; 3416 ed->encryption = BTRFS_ENCRYPTION_NONE; 3417 ed->encoding = BTRFS_ENCODING_NONE; 3418 ed->type = EXTENT_TYPE_INLINE; 3419 3420 Status = read_file(fcb, ed->data, ext->offset, oldalloc, NULL, Irp); 3421 if (!NT_SUCCESS(Status)) { 3422 ERR("read_file returned %08x\n", Status); 3423 ExFreePool(ed); 3424 return Status; 3425 } 3426 3427 RtlZeroMemory(ed->data + oldalloc - ext->offset, (ULONG)(end - oldalloc)); 3428 3429 remove_fcb_extent(fcb, ext, rollback); 3430 3431 Status = add_extent_to_fcb(fcb, ext->offset, ed, edsize, ext->unique, NULL, rollback); 3432 if (!NT_SUCCESS(Status)) { 3433 ERR("add_extent_to_fcb returned %08x\n", Status); 3434 ExFreePool(ed); 3435 return Status; 3436 } 3437 3438 ExFreePool(ed); 3439 3440 fcb->extents_changed = true; 3441 mark_fcb_dirty(fcb); 3442 } 3443 3444 TRACE("extending inline file (oldalloc = %I64x, end = %I64x)\n", oldalloc, end); 3445 3446 fcb->inode_item.st_size = end; 3447 TRACE("setting st_size to %I64x\n", end); 3448 3449 fcb->inode_item.st_blocks = end; 3450 3451 fcb->Header.AllocationSize.QuadPart = fcb->Header.FileSize.QuadPart = fcb->Header.ValidDataLength.QuadPart = end; 3452 } else { 3453 newalloc = sector_align(end, fcb->Vcb->superblock.sector_size); 3454 3455 if (newalloc > oldalloc) { 3456 if (prealloc) { 3457 // FIXME - try and extend previous extent first 3458 3459 Status = insert_prealloc_extent(fcb, oldalloc, newalloc - oldalloc, rollback); 3460 3461 if (!NT_SUCCESS(Status)) { 3462 ERR("insert_prealloc_extent returned %08x\n", Status); 3463 return Status; 3464 } 3465 } 3466 3467 fcb->extents_changed = true; 3468 } 3469 3470 fcb->inode_item.st_size = end; 3471 fcb->inode_item_changed = true; 3472 mark_fcb_dirty(fcb); 3473 3474 TRACE("setting st_size to %I64x\n", end); 3475 3476 TRACE("newalloc = %I64x\n", newalloc); 3477 3478 fcb->Header.AllocationSize.QuadPart = newalloc; 3479 fcb->Header.FileSize.QuadPart = fcb->Header.ValidDataLength.QuadPart = end; 3480 } 3481 } else { 3482 if (end > fcb->Vcb->options.max_inline) { 3483 newalloc = sector_align(end, fcb->Vcb->superblock.sector_size); 3484 3485 if (prealloc) { 3486 Status = insert_prealloc_extent(fcb, 0, newalloc, rollback); 3487 3488 if (!NT_SUCCESS(Status)) { 3489 ERR("insert_prealloc_extent returned %08x\n", Status); 3490 return Status; 3491 } 3492 } 3493 3494 fcb->extents_changed = true; 3495 fcb->inode_item_changed = true; 3496 mark_fcb_dirty(fcb); 3497 3498 fcb->inode_item.st_size = end; 3499 TRACE("setting st_size to %I64x\n", end); 3500 3501 TRACE("newalloc = %I64x\n", newalloc); 3502 3503 fcb->Header.AllocationSize.QuadPart = newalloc; 3504 fcb->Header.FileSize.QuadPart = fcb->Header.ValidDataLength.QuadPart = end; 3505 } else { 3506 EXTENT_DATA* ed; 3507 uint16_t edsize; 3508 3509 edsize = (uint16_t)(offsetof(EXTENT_DATA, data[0]) + end); 3510 ed = ExAllocatePoolWithTag(PagedPool, edsize, ALLOC_TAG); 3511 3512 if (!ed) { 3513 ERR("out of memory\n"); 3514 return STATUS_INSUFFICIENT_RESOURCES; 3515 } 3516 3517 ed->generation = fcb->Vcb->superblock.generation; 3518 ed->decoded_size = end; 3519 ed->compression = BTRFS_COMPRESSION_NONE; 3520 ed->encryption = BTRFS_ENCRYPTION_NONE; 3521 ed->encoding = BTRFS_ENCODING_NONE; 3522 ed->type = EXTENT_TYPE_INLINE; 3523 3524 RtlZeroMemory(ed->data, (ULONG)end); 3525 3526 Status = add_extent_to_fcb(fcb, 0, ed, edsize, false, NULL, rollback); 3527 if (!NT_SUCCESS(Status)) { 3528 ERR("add_extent_to_fcb returned %08x\n", Status); 3529 ExFreePool(ed); 3530 return Status; 3531 } 3532 3533 ExFreePool(ed); 3534 3535 fcb->extents_changed = true; 3536 fcb->inode_item_changed = true; 3537 mark_fcb_dirty(fcb); 3538 3539 fcb->inode_item.st_size = end; 3540 TRACE("setting st_size to %I64x\n", end); 3541 3542 fcb->inode_item.st_blocks = end; 3543 3544 fcb->Header.AllocationSize.QuadPart = fcb->Header.FileSize.QuadPart = fcb->Header.ValidDataLength.QuadPart = end; 3545 } 3546 } 3547 } 3548 3549 return STATUS_SUCCESS; 3550 } 3551 3552 static NTSTATUS do_write_file_prealloc(fcb* fcb, extent* ext, uint64_t start_data, uint64_t end_data, void* data, uint64_t* written, 3553 PIRP Irp, bool file_write, uint64_t irp_offset, ULONG priority, LIST_ENTRY* rollback) { 3554 EXTENT_DATA* ed = &ext->extent_data; 3555 EXTENT_DATA2* ed2 = (EXTENT_DATA2*)ed->data; 3556 NTSTATUS Status; 3557 chunk* c = NULL; 3558 3559 if (start_data <= ext->offset && end_data >= ext->offset + ed2->num_bytes) { // replace all 3560 extent* newext; 3561 3562 newext = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3563 if (!newext) { 3564 ERR("out of memory\n"); 3565 return STATUS_INSUFFICIENT_RESOURCES; 3566 } 3567 3568 RtlCopyMemory(&newext->extent_data, &ext->extent_data, ext->datalen); 3569 3570 newext->extent_data.type = EXTENT_TYPE_REGULAR; 3571 3572 Status = write_data_complete(fcb->Vcb, ed2->address + ed2->offset, (uint8_t*)data + ext->offset - start_data, (uint32_t)ed2->num_bytes, Irp, 3573 NULL, file_write, irp_offset + ext->offset - start_data, priority); 3574 if (!NT_SUCCESS(Status)) { 3575 ERR("write_data_complete returned %08x\n", Status); 3576 return Status; 3577 } 3578 3579 if (!(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 3580 ULONG sl = (ULONG)(ed2->num_bytes / fcb->Vcb->superblock.sector_size); 3581 uint32_t* csum = ExAllocatePoolWithTag(PagedPool, sl * sizeof(uint32_t), ALLOC_TAG); 3582 3583 if (!csum) { 3584 ERR("out of memory\n"); 3585 ExFreePool(newext); 3586 return STATUS_INSUFFICIENT_RESOURCES; 3587 } 3588 3589 Status = calc_csum(fcb->Vcb, (uint8_t*)data + ext->offset - start_data, sl, csum); 3590 if (!NT_SUCCESS(Status)) { 3591 ERR("calc_csum returned %08x\n", Status); 3592 ExFreePool(csum); 3593 ExFreePool(newext); 3594 return Status; 3595 } 3596 3597 newext->csum = csum; 3598 } else 3599 newext->csum = NULL; 3600 3601 *written = ed2->num_bytes; 3602 3603 newext->offset = ext->offset; 3604 newext->datalen = ext->datalen; 3605 newext->unique = ext->unique; 3606 newext->ignore = false; 3607 newext->inserted = true; 3608 InsertHeadList(&ext->list_entry, &newext->list_entry); 3609 3610 add_insert_extent_rollback(rollback, fcb, newext); 3611 3612 remove_fcb_extent(fcb, ext, rollback); 3613 3614 c = get_chunk_from_address(fcb->Vcb, ed2->address); 3615 } else if (start_data <= ext->offset && end_data < ext->offset + ed2->num_bytes) { // replace beginning 3616 EXTENT_DATA2* ned2; 3617 extent *newext1, *newext2; 3618 3619 newext1 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3620 if (!newext1) { 3621 ERR("out of memory\n"); 3622 return STATUS_INSUFFICIENT_RESOURCES; 3623 } 3624 3625 newext2 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3626 if (!newext2) { 3627 ERR("out of memory\n"); 3628 ExFreePool(newext1); 3629 return STATUS_INSUFFICIENT_RESOURCES; 3630 } 3631 3632 RtlCopyMemory(&newext1->extent_data, &ext->extent_data, ext->datalen); 3633 newext1->extent_data.type = EXTENT_TYPE_REGULAR; 3634 ned2 = (EXTENT_DATA2*)newext1->extent_data.data; 3635 ned2->num_bytes = end_data - ext->offset; 3636 3637 RtlCopyMemory(&newext2->extent_data, &ext->extent_data, ext->datalen); 3638 ned2 = (EXTENT_DATA2*)newext2->extent_data.data; 3639 ned2->offset += end_data - ext->offset; 3640 ned2->num_bytes -= end_data - ext->offset; 3641 3642 Status = write_data_complete(fcb->Vcb, ed2->address + ed2->offset, (uint8_t*)data + ext->offset - start_data, (uint32_t)(end_data - ext->offset), 3643 Irp, NULL, file_write, irp_offset + ext->offset - start_data, priority); 3644 if (!NT_SUCCESS(Status)) { 3645 ERR("write_data_complete returned %08x\n", Status); 3646 ExFreePool(newext1); 3647 ExFreePool(newext2); 3648 return Status; 3649 } 3650 3651 if (!(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 3652 ULONG sl = (ULONG)((end_data - ext->offset) / fcb->Vcb->superblock.sector_size); 3653 uint32_t* csum = ExAllocatePoolWithTag(PagedPool, sl * sizeof(uint32_t), ALLOC_TAG); 3654 3655 if (!csum) { 3656 ERR("out of memory\n"); 3657 ExFreePool(newext1); 3658 ExFreePool(newext2); 3659 return STATUS_INSUFFICIENT_RESOURCES; 3660 } 3661 3662 Status = calc_csum(fcb->Vcb, (uint8_t*)data + ext->offset - start_data, sl, csum); 3663 if (!NT_SUCCESS(Status)) { 3664 ERR("calc_csum returned %08x\n", Status); 3665 ExFreePool(newext1); 3666 ExFreePool(newext2); 3667 ExFreePool(csum); 3668 return Status; 3669 } 3670 3671 newext1->csum = csum; 3672 } else 3673 newext1->csum = NULL; 3674 3675 *written = end_data - ext->offset; 3676 3677 newext1->offset = ext->offset; 3678 newext1->datalen = ext->datalen; 3679 newext1->unique = ext->unique; 3680 newext1->ignore = false; 3681 newext1->inserted = true; 3682 InsertHeadList(&ext->list_entry, &newext1->list_entry); 3683 3684 add_insert_extent_rollback(rollback, fcb, newext1); 3685 3686 newext2->offset = end_data; 3687 newext2->datalen = ext->datalen; 3688 newext2->unique = ext->unique; 3689 newext2->ignore = false; 3690 newext2->inserted = true; 3691 newext2->csum = NULL; 3692 add_extent(fcb, &newext1->list_entry, newext2); 3693 3694 add_insert_extent_rollback(rollback, fcb, newext2); 3695 3696 c = get_chunk_from_address(fcb->Vcb, ed2->address); 3697 3698 if (!c) 3699 ERR("get_chunk_from_address(%I64x) failed\n", ed2->address); 3700 else { 3701 Status = update_changed_extent_ref(fcb->Vcb, c, ed2->address, ed2->size, fcb->subvol->id, fcb->inode, ext->offset - ed2->offset, 1, 3702 fcb->inode_item.flags & BTRFS_INODE_NODATASUM, false, Irp); 3703 3704 if (!NT_SUCCESS(Status)) { 3705 ERR("update_changed_extent_ref returned %08x\n", Status); 3706 return Status; 3707 } 3708 } 3709 3710 remove_fcb_extent(fcb, ext, rollback); 3711 } else if (start_data > ext->offset && end_data >= ext->offset + ed2->num_bytes) { // replace end 3712 EXTENT_DATA2* ned2; 3713 extent *newext1, *newext2; 3714 3715 newext1 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3716 if (!newext1) { 3717 ERR("out of memory\n"); 3718 return STATUS_INSUFFICIENT_RESOURCES; 3719 } 3720 3721 newext2 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3722 if (!newext2) { 3723 ERR("out of memory\n"); 3724 ExFreePool(newext1); 3725 return STATUS_INSUFFICIENT_RESOURCES; 3726 } 3727 3728 RtlCopyMemory(&newext1->extent_data, &ext->extent_data, ext->datalen); 3729 3730 ned2 = (EXTENT_DATA2*)newext1->extent_data.data; 3731 ned2->num_bytes = start_data - ext->offset; 3732 3733 RtlCopyMemory(&newext2->extent_data, &ext->extent_data, ext->datalen); 3734 3735 newext2->extent_data.type = EXTENT_TYPE_REGULAR; 3736 ned2 = (EXTENT_DATA2*)newext2->extent_data.data; 3737 ned2->offset += start_data - ext->offset; 3738 ned2->num_bytes = ext->offset + ed2->num_bytes - start_data; 3739 3740 Status = write_data_complete(fcb->Vcb, ed2->address + ned2->offset, data, (uint32_t)ned2->num_bytes, Irp, NULL, file_write, irp_offset, priority); 3741 if (!NT_SUCCESS(Status)) { 3742 ERR("write_data_complete returned %08x\n", Status); 3743 ExFreePool(newext1); 3744 ExFreePool(newext2); 3745 return Status; 3746 } 3747 3748 if (!(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 3749 ULONG sl = (ULONG)(ned2->num_bytes / fcb->Vcb->superblock.sector_size); 3750 uint32_t* csum = ExAllocatePoolWithTag(PagedPool, sl * sizeof(uint32_t), ALLOC_TAG); 3751 3752 if (!csum) { 3753 ERR("out of memory\n"); 3754 ExFreePool(newext1); 3755 ExFreePool(newext2); 3756 return STATUS_INSUFFICIENT_RESOURCES; 3757 } 3758 3759 Status = calc_csum(fcb->Vcb, data, sl, csum); 3760 if (!NT_SUCCESS(Status)) { 3761 ERR("calc_csum returned %08x\n", Status); 3762 ExFreePool(newext1); 3763 ExFreePool(newext2); 3764 ExFreePool(csum); 3765 return Status; 3766 } 3767 3768 newext2->csum = csum; 3769 } else 3770 newext2->csum = NULL; 3771 3772 *written = ned2->num_bytes; 3773 3774 newext1->offset = ext->offset; 3775 newext1->datalen = ext->datalen; 3776 newext1->unique = ext->unique; 3777 newext1->ignore = false; 3778 newext1->inserted = true; 3779 newext1->csum = NULL; 3780 InsertHeadList(&ext->list_entry, &newext1->list_entry); 3781 3782 add_insert_extent_rollback(rollback, fcb, newext1); 3783 3784 newext2->offset = start_data; 3785 newext2->datalen = ext->datalen; 3786 newext2->unique = ext->unique; 3787 newext2->ignore = false; 3788 newext2->inserted = true; 3789 add_extent(fcb, &newext1->list_entry, newext2); 3790 3791 add_insert_extent_rollback(rollback, fcb, newext2); 3792 3793 c = get_chunk_from_address(fcb->Vcb, ed2->address); 3794 3795 if (!c) 3796 ERR("get_chunk_from_address(%I64x) failed\n", ed2->address); 3797 else { 3798 Status = update_changed_extent_ref(fcb->Vcb, c, ed2->address, ed2->size, fcb->subvol->id, fcb->inode, ext->offset - ed2->offset, 1, 3799 fcb->inode_item.flags & BTRFS_INODE_NODATASUM, false, Irp); 3800 3801 if (!NT_SUCCESS(Status)) { 3802 ERR("update_changed_extent_ref returned %08x\n", Status); 3803 return Status; 3804 } 3805 } 3806 3807 remove_fcb_extent(fcb, ext, rollback); 3808 } else if (start_data > ext->offset && end_data < ext->offset + ed2->num_bytes) { // replace middle 3809 EXTENT_DATA2* ned2; 3810 extent *newext1, *newext2, *newext3; 3811 3812 newext1 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3813 if (!newext1) { 3814 ERR("out of memory\n"); 3815 return STATUS_INSUFFICIENT_RESOURCES; 3816 } 3817 3818 newext2 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3819 if (!newext2) { 3820 ERR("out of memory\n"); 3821 ExFreePool(newext1); 3822 return STATUS_INSUFFICIENT_RESOURCES; 3823 } 3824 3825 newext3 = ExAllocatePoolWithTag(PagedPool, offsetof(extent, extent_data) + ext->datalen, ALLOC_TAG); 3826 if (!newext3) { 3827 ERR("out of memory\n"); 3828 ExFreePool(newext1); 3829 ExFreePool(newext2); 3830 return STATUS_INSUFFICIENT_RESOURCES; 3831 } 3832 3833 RtlCopyMemory(&newext1->extent_data, &ext->extent_data, ext->datalen); 3834 RtlCopyMemory(&newext2->extent_data, &ext->extent_data, ext->datalen); 3835 RtlCopyMemory(&newext3->extent_data, &ext->extent_data, ext->datalen); 3836 3837 ned2 = (EXTENT_DATA2*)newext1->extent_data.data; 3838 ned2->num_bytes = start_data - ext->offset; 3839 3840 newext2->extent_data.type = EXTENT_TYPE_REGULAR; 3841 ned2 = (EXTENT_DATA2*)newext2->extent_data.data; 3842 ned2->offset += start_data - ext->offset; 3843 ned2->num_bytes = end_data - start_data; 3844 3845 ned2 = (EXTENT_DATA2*)newext3->extent_data.data; 3846 ned2->offset += end_data - ext->offset; 3847 ned2->num_bytes -= end_data - ext->offset; 3848 3849 ned2 = (EXTENT_DATA2*)newext2->extent_data.data; 3850 Status = write_data_complete(fcb->Vcb, ed2->address + ned2->offset, data, (uint32_t)(end_data - start_data), Irp, NULL, file_write, irp_offset, priority); 3851 if (!NT_SUCCESS(Status)) { 3852 ERR("write_data_complete returned %08x\n", Status); 3853 ExFreePool(newext1); 3854 ExFreePool(newext2); 3855 ExFreePool(newext3); 3856 return Status; 3857 } 3858 3859 if (!(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 3860 ULONG sl = (ULONG)((end_data - start_data) / fcb->Vcb->superblock.sector_size); 3861 uint32_t* csum = ExAllocatePoolWithTag(PagedPool, sl * sizeof(uint32_t), ALLOC_TAG); 3862 3863 if (!csum) { 3864 ERR("out of memory\n"); 3865 ExFreePool(newext1); 3866 ExFreePool(newext2); 3867 ExFreePool(newext3); 3868 return STATUS_INSUFFICIENT_RESOURCES; 3869 } 3870 3871 Status = calc_csum(fcb->Vcb, data, sl, csum); 3872 if (!NT_SUCCESS(Status)) { 3873 ERR("calc_csum returned %08x\n", Status); 3874 ExFreePool(newext1); 3875 ExFreePool(newext2); 3876 ExFreePool(newext3); 3877 ExFreePool(csum); 3878 return Status; 3879 } 3880 3881 newext2->csum = csum; 3882 } else 3883 newext2->csum = NULL; 3884 3885 *written = end_data - start_data; 3886 3887 newext1->offset = ext->offset; 3888 newext1->datalen = ext->datalen; 3889 newext1->unique = ext->unique; 3890 newext1->ignore = false; 3891 newext1->inserted = true; 3892 newext1->csum = NULL; 3893 InsertHeadList(&ext->list_entry, &newext1->list_entry); 3894 3895 add_insert_extent_rollback(rollback, fcb, newext1); 3896 3897 newext2->offset = start_data; 3898 newext2->datalen = ext->datalen; 3899 newext2->unique = ext->unique; 3900 newext2->ignore = false; 3901 newext2->inserted = true; 3902 add_extent(fcb, &newext1->list_entry, newext2); 3903 3904 add_insert_extent_rollback(rollback, fcb, newext2); 3905 3906 newext3->offset = end_data; 3907 newext3->datalen = ext->datalen; 3908 newext3->unique = ext->unique; 3909 newext3->ignore = false; 3910 newext3->inserted = true; 3911 newext3->csum = NULL; 3912 add_extent(fcb, &newext2->list_entry, newext3); 3913 3914 add_insert_extent_rollback(rollback, fcb, newext3); 3915 3916 c = get_chunk_from_address(fcb->Vcb, ed2->address); 3917 3918 if (!c) 3919 ERR("get_chunk_from_address(%I64x) failed\n", ed2->address); 3920 else { 3921 Status = update_changed_extent_ref(fcb->Vcb, c, ed2->address, ed2->size, fcb->subvol->id, fcb->inode, ext->offset - ed2->offset, 2, 3922 fcb->inode_item.flags & BTRFS_INODE_NODATASUM, false, Irp); 3923 3924 if (!NT_SUCCESS(Status)) { 3925 ERR("update_changed_extent_ref returned %08x\n", Status); 3926 return Status; 3927 } 3928 } 3929 3930 remove_fcb_extent(fcb, ext, rollback); 3931 } 3932 3933 if (c) 3934 c->changed = true; 3935 3936 return STATUS_SUCCESS; 3937 } 3938 3939 NTSTATUS do_write_file(fcb* fcb, uint64_t start, uint64_t end_data, void* data, PIRP Irp, bool file_write, uint32_t irp_offset, LIST_ENTRY* rollback) { 3940 NTSTATUS Status; 3941 LIST_ENTRY *le, *le2; 3942 uint64_t written = 0, length = end_data - start; 3943 uint64_t last_cow_start; 3944 ULONG priority = fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE ? HighPagePriority : NormalPagePriority; 3945 #ifdef DEBUG_PARANOID 3946 uint64_t last_off; 3947 #endif 3948 bool extents_changed = false; 3949 3950 last_cow_start = 0; 3951 3952 le = fcb->extents.Flink; 3953 while (le != &fcb->extents) { 3954 extent* ext = CONTAINING_RECORD(le, extent, list_entry); 3955 3956 le2 = le->Flink; 3957 3958 if (!ext->ignore) { 3959 EXTENT_DATA* ed = &ext->extent_data; 3960 EXTENT_DATA2* ed2 = ed->type == EXTENT_TYPE_INLINE ? NULL : (EXTENT_DATA2*)ed->data; 3961 uint64_t len; 3962 3963 len = ed->type == EXTENT_TYPE_INLINE ? ed->decoded_size : ed2->num_bytes; 3964 3965 if (ext->offset + len <= start) 3966 goto nextitem; 3967 3968 if (ext->offset > start + written + length) 3969 break; 3970 3971 if ((fcb->inode_item.flags & BTRFS_INODE_NODATACOW || ed->type == EXTENT_TYPE_PREALLOC) && ext->unique && ed->compression == BTRFS_COMPRESSION_NONE) { 3972 if (max(last_cow_start, start + written) < ext->offset) { 3973 uint64_t start_write = max(last_cow_start, start + written); 3974 3975 extents_changed = true; 3976 3977 Status = excise_extents(fcb->Vcb, fcb, start_write, ext->offset, Irp, rollback); 3978 if (!NT_SUCCESS(Status)) { 3979 ERR("excise_extents returned %08x\n", Status); 3980 return Status; 3981 } 3982 3983 Status = insert_extent(fcb->Vcb, fcb, start_write, ext->offset - start_write, (uint8_t*)data + written, Irp, file_write, irp_offset + written, rollback); 3984 if (!NT_SUCCESS(Status)) { 3985 ERR("insert_extent returned %08x\n", Status); 3986 return Status; 3987 } 3988 3989 written += ext->offset - start_write; 3990 length -= ext->offset - start_write; 3991 3992 if (length == 0) 3993 break; 3994 } 3995 3996 if (ed->type == EXTENT_TYPE_REGULAR) { 3997 uint64_t writeaddr = ed2->address + ed2->offset + start + written - ext->offset; 3998 uint64_t write_len = min(len, length); 3999 chunk* c; 4000 4001 TRACE("doing non-COW write to %I64x\n", writeaddr); 4002 4003 Status = write_data_complete(fcb->Vcb, writeaddr, (uint8_t*)data + written, (uint32_t)write_len, Irp, NULL, file_write, irp_offset + written, priority); 4004 if (!NT_SUCCESS(Status)) { 4005 ERR("write_data_complete returned %08x\n", Status); 4006 return Status; 4007 } 4008 4009 c = get_chunk_from_address(fcb->Vcb, writeaddr); 4010 if (c) 4011 c->changed = true; 4012 4013 // This shouldn't ever get called - nocow files should always also be nosum. 4014 if (!(fcb->inode_item.flags & BTRFS_INODE_NODATASUM)) { 4015 calc_csum(fcb->Vcb, (uint8_t*)data + written, (uint32_t)(write_len / fcb->Vcb->superblock.sector_size), 4016 &ext->csum[(start + written - ext->offset) / fcb->Vcb->superblock.sector_size]); 4017 4018 ext->inserted = true; 4019 extents_changed = true; 4020 } 4021 4022 written += write_len; 4023 length -= write_len; 4024 4025 if (length == 0) 4026 break; 4027 } else if (ed->type == EXTENT_TYPE_PREALLOC) { 4028 uint64_t write_len; 4029 4030 Status = do_write_file_prealloc(fcb, ext, start + written, end_data, (uint8_t*)data + written, &write_len, 4031 Irp, file_write, irp_offset + written, priority, rollback); 4032 if (!NT_SUCCESS(Status)) { 4033 ERR("do_write_file_prealloc returned %08x\n", Status); 4034 return Status; 4035 } 4036 4037 extents_changed = true; 4038 4039 written += write_len; 4040 length -= write_len; 4041 4042 if (length == 0) 4043 break; 4044 } 4045 4046 last_cow_start = ext->offset + len; 4047 } 4048 } 4049 4050 nextitem: 4051 le = le2; 4052 } 4053 4054 if (length > 0) { 4055 uint64_t start_write = max(last_cow_start, start + written); 4056 4057 extents_changed = true; 4058 4059 Status = excise_extents(fcb->Vcb, fcb, start_write, end_data, Irp, rollback); 4060 if (!NT_SUCCESS(Status)) { 4061 ERR("excise_extents returned %08x\n", Status); 4062 return Status; 4063 } 4064 4065 Status = insert_extent(fcb->Vcb, fcb, start_write, end_data - start_write, (uint8_t*)data + written, Irp, file_write, irp_offset + written, rollback); 4066 if (!NT_SUCCESS(Status)) { 4067 ERR("insert_extent returned %08x\n", Status); 4068 return Status; 4069 } 4070 } 4071 4072 #ifdef DEBUG_PARANOID 4073 last_off = 0xffffffffffffffff; 4074 4075 le = fcb->extents.Flink; 4076 while (le != &fcb->extents) { 4077 extent* ext = CONTAINING_RECORD(le, extent, list_entry); 4078 4079 if (!ext->ignore) { 4080 if (ext->offset == last_off) { 4081 ERR("offset %I64x duplicated\n", ext->offset); 4082 int3; 4083 } else if (ext->offset < last_off && last_off != 0xffffffffffffffff) { 4084 ERR("offsets out of order\n"); 4085 int3; 4086 } 4087 4088 last_off = ext->offset; 4089 } 4090 4091 le = le->Flink; 4092 } 4093 #endif 4094 4095 if (extents_changed) { 4096 fcb->extents_changed = true; 4097 mark_fcb_dirty(fcb); 4098 } 4099 4100 return STATUS_SUCCESS; 4101 } 4102 4103 NTSTATUS write_compressed(fcb* fcb, uint64_t start_data, uint64_t end_data, void* data, PIRP Irp, LIST_ENTRY* rollback) { 4104 NTSTATUS Status; 4105 uint64_t i; 4106 4107 for (i = 0; i < sector_align(end_data - start_data, COMPRESSED_EXTENT_SIZE) / COMPRESSED_EXTENT_SIZE; i++) { 4108 uint64_t s2, e2; 4109 bool compressed; 4110 4111 s2 = start_data + (i * COMPRESSED_EXTENT_SIZE); 4112 e2 = min(s2 + COMPRESSED_EXTENT_SIZE, end_data); 4113 4114 Status = write_compressed_bit(fcb, s2, e2, (uint8_t*)data + (i * COMPRESSED_EXTENT_SIZE), &compressed, Irp, rollback); 4115 4116 if (!NT_SUCCESS(Status)) { 4117 ERR("write_compressed_bit returned %08x\n", Status); 4118 return Status; 4119 } 4120 4121 // If the first 128 KB of a file is incompressible, we set the nocompress flag so we don't 4122 // bother with the rest of it. 4123 if (s2 == 0 && e2 == COMPRESSED_EXTENT_SIZE && !compressed && !fcb->Vcb->options.compress_force) { 4124 fcb->inode_item.flags |= BTRFS_INODE_NOCOMPRESS; 4125 fcb->inode_item_changed = true; 4126 mark_fcb_dirty(fcb); 4127 4128 // write subsequent data non-compressed 4129 if (e2 < end_data) { 4130 Status = do_write_file(fcb, e2, end_data, (uint8_t*)data + e2, Irp, false, 0, rollback); 4131 4132 if (!NT_SUCCESS(Status)) { 4133 ERR("do_write_file returned %08x\n", Status); 4134 return Status; 4135 } 4136 } 4137 4138 return STATUS_SUCCESS; 4139 } 4140 } 4141 4142 return STATUS_SUCCESS; 4143 } 4144 4145 NTSTATUS write_file2(device_extension* Vcb, PIRP Irp, LARGE_INTEGER offset, void* buf, ULONG* length, bool paging_io, bool no_cache, 4146 bool wait, bool deferred_write, bool write_irp, LIST_ENTRY* rollback) { 4147 PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp); 4148 PFILE_OBJECT FileObject = IrpSp->FileObject; 4149 EXTENT_DATA* ed2; 4150 uint64_t off64, newlength, start_data, end_data; 4151 uint32_t bufhead; 4152 bool make_inline; 4153 INODE_ITEM* origii; 4154 bool changed_length = false; 4155 NTSTATUS Status; 4156 LARGE_INTEGER time; 4157 BTRFS_TIME now; 4158 fcb* fcb; 4159 ccb* ccb; 4160 file_ref* fileref; 4161 bool paging_lock = false, acquired_fcb_lock = false, acquired_tree_lock = false, pagefile; 4162 ULONG filter = 0; 4163 4164 TRACE("(%p, %p, %I64x, %p, %x, %u, %u)\n", Vcb, FileObject, offset.QuadPart, buf, *length, paging_io, no_cache); 4165 4166 if (*length == 0) { 4167 TRACE("returning success for zero-length write\n"); 4168 return STATUS_SUCCESS; 4169 } 4170 4171 if (!FileObject) { 4172 ERR("error - FileObject was NULL\n"); 4173 return STATUS_ACCESS_DENIED; 4174 } 4175 4176 fcb = FileObject->FsContext; 4177 ccb = FileObject->FsContext2; 4178 fileref = ccb ? ccb->fileref : NULL; 4179 4180 if (!fcb->ads && fcb->type != BTRFS_TYPE_FILE && fcb->type != BTRFS_TYPE_SYMLINK) { 4181 WARN("tried to write to something other than a file or symlink (inode %I64x, type %u, %p, %p)\n", fcb->inode, fcb->type, &fcb->type, fcb); 4182 return STATUS_INVALID_DEVICE_REQUEST; 4183 } 4184 4185 if (offset.LowPart == FILE_WRITE_TO_END_OF_FILE && offset.HighPart == -1) 4186 offset = fcb->Header.FileSize; 4187 4188 off64 = offset.QuadPart; 4189 4190 TRACE("fcb->Header.Flags = %x\n", fcb->Header.Flags); 4191 4192 if (!no_cache && !CcCanIWrite(FileObject, *length, wait, deferred_write)) 4193 return STATUS_PENDING; 4194 4195 if (!wait && no_cache) 4196 return STATUS_PENDING; 4197 4198 if (no_cache && !paging_io && FileObject->SectionObjectPointer->DataSectionObject) { 4199 IO_STATUS_BLOCK iosb; 4200 4201 ExAcquireResourceExclusiveLite(fcb->Header.PagingIoResource, true); 4202 4203 CcFlushCache(FileObject->SectionObjectPointer, &offset, *length, &iosb); 4204 4205 if (!NT_SUCCESS(iosb.Status)) { 4206 ExReleaseResourceLite(fcb->Header.PagingIoResource); 4207 ERR("CcFlushCache returned %08x\n", iosb.Status); 4208 return iosb.Status; 4209 } 4210 4211 paging_lock = true; 4212 4213 CcPurgeCacheSection(FileObject->SectionObjectPointer, &offset, *length, false); 4214 } 4215 4216 if (paging_io) { 4217 if (!ExAcquireResourceSharedLite(fcb->Header.PagingIoResource, wait)) { 4218 Status = STATUS_PENDING; 4219 goto end; 4220 } else 4221 paging_lock = true; 4222 } 4223 4224 pagefile = fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE && paging_io; 4225 4226 if (!pagefile && !ExIsResourceAcquiredExclusiveLite(&Vcb->tree_lock)) { 4227 if (!ExAcquireResourceSharedLite(&Vcb->tree_lock, wait)) { 4228 Status = STATUS_PENDING; 4229 goto end; 4230 } else 4231 acquired_tree_lock = true; 4232 } 4233 4234 if (pagefile) { 4235 if (!ExAcquireResourceSharedLite(fcb->Header.Resource, wait)) { 4236 Status = STATUS_PENDING; 4237 goto end; 4238 } else 4239 acquired_fcb_lock = true; 4240 } else if (!ExIsResourceAcquiredExclusiveLite(fcb->Header.Resource)) { 4241 if (!ExAcquireResourceExclusiveLite(fcb->Header.Resource, wait)) { 4242 Status = STATUS_PENDING; 4243 goto end; 4244 } else 4245 acquired_fcb_lock = true; 4246 } 4247 4248 newlength = fcb->ads ? fcb->adsdata.Length : fcb->inode_item.st_size; 4249 4250 if (fcb->deleted) 4251 newlength = 0; 4252 4253 TRACE("newlength = %I64x\n", newlength); 4254 4255 if (off64 + *length > newlength) { 4256 if (paging_io) { 4257 if (off64 >= newlength) { 4258 TRACE("paging IO tried to write beyond end of file (file size = %I64x, offset = %I64x, length = %x)\n", newlength, off64, *length); 4259 TRACE("FileObject: AllocationSize = %I64x, FileSize = %I64x, ValidDataLength = %I64x\n", 4260 fcb->Header.AllocationSize.QuadPart, fcb->Header.FileSize.QuadPart, fcb->Header.ValidDataLength.QuadPart); 4261 Status = STATUS_SUCCESS; 4262 goto end; 4263 } 4264 4265 *length = (ULONG)(newlength - off64); 4266 } else { 4267 newlength = off64 + *length; 4268 changed_length = true; 4269 4270 TRACE("extending length to %I64x\n", newlength); 4271 } 4272 } 4273 4274 if (fcb->ads) 4275 make_inline = false; 4276 else if (fcb->type == BTRFS_TYPE_SYMLINK) 4277 make_inline = newlength <= (Vcb->superblock.node_size - sizeof(tree_header) - sizeof(leaf_node) - offsetof(EXTENT_DATA, data[0])); 4278 else 4279 make_inline = newlength <= fcb->Vcb->options.max_inline; 4280 4281 if (changed_length) { 4282 if (newlength > (uint64_t)fcb->Header.AllocationSize.QuadPart) { 4283 if (!acquired_tree_lock) { 4284 // We need to acquire the tree lock if we don't have it already - 4285 // we can't give an inline file proper extents at the same time as we're 4286 // doing a flush. 4287 if (!ExAcquireResourceSharedLite(&Vcb->tree_lock, wait)) { 4288 Status = STATUS_PENDING; 4289 goto end; 4290 } else 4291 acquired_tree_lock = true; 4292 } 4293 4294 Status = extend_file(fcb, fileref, newlength, false, Irp, rollback); 4295 if (!NT_SUCCESS(Status)) { 4296 ERR("extend_file returned %08x\n", Status); 4297 goto end; 4298 } 4299 } else if (!fcb->ads) 4300 fcb->inode_item.st_size = newlength; 4301 4302 fcb->Header.FileSize.QuadPart = newlength; 4303 fcb->Header.ValidDataLength.QuadPart = newlength; 4304 4305 TRACE("AllocationSize = %I64x\n", fcb->Header.AllocationSize.QuadPart); 4306 TRACE("FileSize = %I64x\n", fcb->Header.FileSize.QuadPart); 4307 TRACE("ValidDataLength = %I64x\n", fcb->Header.ValidDataLength.QuadPart); 4308 } 4309 4310 if (!no_cache) { 4311 Status = STATUS_SUCCESS; 4312 4313 _SEH2_TRY { 4314 if (!FileObject->PrivateCacheMap || changed_length) { 4315 CC_FILE_SIZES ccfs; 4316 4317 ccfs.AllocationSize = fcb->Header.AllocationSize; 4318 ccfs.FileSize = fcb->Header.FileSize; 4319 ccfs.ValidDataLength = fcb->Header.ValidDataLength; 4320 4321 if (!FileObject->PrivateCacheMap) 4322 init_file_cache(FileObject, &ccfs); 4323 4324 CcSetFileSizes(FileObject, &ccfs); 4325 } 4326 4327 if (IrpSp->MinorFunction & IRP_MN_MDL) { 4328 CcPrepareMdlWrite(FileObject, &offset, *length, &Irp->MdlAddress, &Irp->IoStatus); 4329 4330 Status = Irp->IoStatus.Status; 4331 goto end; 4332 } else { 4333 if (fCcCopyWriteEx) { 4334 TRACE("CcCopyWriteEx(%p, %I64x, %x, %u, %p, %p)\n", FileObject, off64, *length, wait, buf, Irp->Tail.Overlay.Thread); 4335 if (!fCcCopyWriteEx(FileObject, &offset, *length, wait, buf, Irp->Tail.Overlay.Thread)) { 4336 Status = STATUS_PENDING; 4337 goto end; 4338 } 4339 TRACE("CcCopyWriteEx finished\n"); 4340 } else { 4341 TRACE("CcCopyWrite(%p, %I64x, %x, %u, %p)\n", FileObject, off64, *length, wait, buf); 4342 if (!CcCopyWrite(FileObject, &offset, *length, wait, buf)) { 4343 Status = STATUS_PENDING; 4344 goto end; 4345 } 4346 TRACE("CcCopyWrite finished\n"); 4347 } 4348 } 4349 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4350 Status = _SEH2_GetExceptionCode(); 4351 } _SEH2_END; 4352 4353 if (changed_length) { 4354 queue_notification_fcb(fcb->ads ? fileref->parent : fileref, fcb->ads ? FILE_NOTIFY_CHANGE_STREAM_SIZE : FILE_NOTIFY_CHANGE_SIZE, 4355 fcb->ads ? FILE_ACTION_MODIFIED_STREAM : FILE_ACTION_MODIFIED, fcb->ads && fileref->dc ? &fileref->dc->name : NULL); 4356 } 4357 4358 goto end; 4359 } 4360 4361 if (fcb->ads) { 4362 if (changed_length) { 4363 char* data2; 4364 4365 if (newlength > fcb->adsmaxlen) { 4366 ERR("error - xattr too long (%I64u > %u)\n", newlength, fcb->adsmaxlen); 4367 Status = STATUS_DISK_FULL; 4368 goto end; 4369 } 4370 4371 data2 = ExAllocatePoolWithTag(PagedPool, (ULONG)newlength, ALLOC_TAG); 4372 if (!data2) { 4373 ERR("out of memory\n"); 4374 Status = STATUS_INSUFFICIENT_RESOURCES; 4375 goto end; 4376 } 4377 4378 if (fcb->adsdata.Buffer) { 4379 RtlCopyMemory(data2, fcb->adsdata.Buffer, fcb->adsdata.Length); 4380 ExFreePool(fcb->adsdata.Buffer); 4381 } 4382 4383 if (newlength > fcb->adsdata.Length) 4384 RtlZeroMemory(&data2[fcb->adsdata.Length], (ULONG)(newlength - fcb->adsdata.Length)); 4385 4386 4387 fcb->adsdata.Buffer = data2; 4388 fcb->adsdata.Length = fcb->adsdata.MaximumLength = (USHORT)newlength; 4389 4390 fcb->Header.AllocationSize.QuadPart = newlength; 4391 fcb->Header.FileSize.QuadPart = newlength; 4392 fcb->Header.ValidDataLength.QuadPart = newlength; 4393 } 4394 4395 if (*length > 0) 4396 RtlCopyMemory(&fcb->adsdata.Buffer[off64], buf, *length); 4397 4398 fcb->Header.ValidDataLength.QuadPart = newlength; 4399 4400 mark_fcb_dirty(fcb); 4401 4402 if (fileref) 4403 mark_fileref_dirty(fileref); 4404 } else { 4405 bool compress = write_fcb_compressed(fcb), no_buf = false; 4406 uint8_t* data; 4407 4408 if (make_inline) { 4409 start_data = 0; 4410 end_data = sector_align(newlength, fcb->Vcb->superblock.sector_size); 4411 bufhead = sizeof(EXTENT_DATA) - 1; 4412 } else if (compress) { 4413 start_data = off64 & ~(uint64_t)(COMPRESSED_EXTENT_SIZE - 1); 4414 end_data = min(sector_align(off64 + *length, COMPRESSED_EXTENT_SIZE), 4415 sector_align(newlength, fcb->Vcb->superblock.sector_size)); 4416 bufhead = 0; 4417 } else { 4418 start_data = off64 & ~(uint64_t)(fcb->Vcb->superblock.sector_size - 1); 4419 end_data = sector_align(off64 + *length, fcb->Vcb->superblock.sector_size); 4420 bufhead = 0; 4421 } 4422 4423 if (fcb_is_inline(fcb)) 4424 end_data = max(end_data, sector_align(fcb->inode_item.st_size, Vcb->superblock.sector_size)); 4425 4426 fcb->Header.ValidDataLength.QuadPart = newlength; 4427 TRACE("fcb %p FileSize = %I64x\n", fcb, fcb->Header.FileSize.QuadPart); 4428 4429 if (!make_inline && !compress && off64 == start_data && off64 + *length == end_data) { 4430 data = buf; 4431 no_buf = true; 4432 } else { 4433 data = ExAllocatePoolWithTag(PagedPool, (ULONG)(end_data - start_data + bufhead), ALLOC_TAG); 4434 if (!data) { 4435 ERR("out of memory\n"); 4436 Status = STATUS_INSUFFICIENT_RESOURCES; 4437 goto end; 4438 } 4439 4440 RtlZeroMemory(data + bufhead, (ULONG)(end_data - start_data)); 4441 4442 TRACE("start_data = %I64x\n", start_data); 4443 TRACE("end_data = %I64x\n", end_data); 4444 4445 if (off64 > start_data || off64 + *length < end_data) { 4446 if (changed_length) { 4447 if (fcb->inode_item.st_size > start_data) 4448 Status = read_file(fcb, data + bufhead, start_data, fcb->inode_item.st_size - start_data, NULL, Irp); 4449 else 4450 Status = STATUS_SUCCESS; 4451 } else 4452 Status = read_file(fcb, data + bufhead, start_data, end_data - start_data, NULL, Irp); 4453 4454 if (!NT_SUCCESS(Status)) { 4455 ERR("read_file returned %08x\n", Status); 4456 ExFreePool(data); 4457 goto end; 4458 } 4459 } 4460 4461 RtlCopyMemory(data + bufhead + off64 - start_data, buf, *length); 4462 } 4463 4464 if (make_inline) { 4465 Status = excise_extents(fcb->Vcb, fcb, start_data, end_data, Irp, rollback); 4466 if (!NT_SUCCESS(Status)) { 4467 ERR("error - excise_extents returned %08x\n", Status); 4468 ExFreePool(data); 4469 goto end; 4470 } 4471 4472 ed2 = (EXTENT_DATA*)data; 4473 ed2->generation = fcb->Vcb->superblock.generation; 4474 ed2->decoded_size = newlength; 4475 ed2->compression = BTRFS_COMPRESSION_NONE; 4476 ed2->encryption = BTRFS_ENCRYPTION_NONE; 4477 ed2->encoding = BTRFS_ENCODING_NONE; 4478 ed2->type = EXTENT_TYPE_INLINE; 4479 4480 Status = add_extent_to_fcb(fcb, 0, ed2, (uint16_t)(offsetof(EXTENT_DATA, data[0]) + newlength), false, NULL, rollback); 4481 if (!NT_SUCCESS(Status)) { 4482 ERR("add_extent_to_fcb returned %08x\n", Status); 4483 ExFreePool(data); 4484 goto end; 4485 } 4486 4487 fcb->inode_item.st_blocks += newlength; 4488 } else if (compress) { 4489 Status = write_compressed(fcb, start_data, end_data, data, Irp, rollback); 4490 4491 if (!NT_SUCCESS(Status)) { 4492 ERR("write_compressed returned %08x\n", Status); 4493 ExFreePool(data); 4494 goto end; 4495 } 4496 } else { 4497 if (write_irp && Irp->MdlAddress && no_buf) { 4498 bool locked = Irp->MdlAddress->MdlFlags & (MDL_PAGES_LOCKED | MDL_PARTIAL); 4499 4500 if (!locked) { 4501 Status = STATUS_SUCCESS; 4502 4503 _SEH2_TRY { 4504 MmProbeAndLockPages(Irp->MdlAddress, KernelMode, IoReadAccess); 4505 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4506 Status = _SEH2_GetExceptionCode(); 4507 } _SEH2_END; 4508 4509 if (!NT_SUCCESS(Status)) { 4510 ERR("MmProbeAndLockPages threw exception %08x\n", Status); 4511 goto end; 4512 } 4513 } 4514 4515 _SEH2_TRY { 4516 Status = do_write_file(fcb, start_data, end_data, data, Irp, true, 0, rollback); 4517 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4518 Status = _SEH2_GetExceptionCode(); 4519 } _SEH2_END; 4520 4521 if (!locked) 4522 MmUnlockPages(Irp->MdlAddress); 4523 } else { 4524 _SEH2_TRY { 4525 Status = do_write_file(fcb, start_data, end_data, data, Irp, false, 0, rollback); 4526 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4527 Status = _SEH2_GetExceptionCode(); 4528 } _SEH2_END; 4529 } 4530 4531 if (!NT_SUCCESS(Status)) { 4532 ERR("do_write_file returned %08x\n", Status); 4533 if (!no_buf) ExFreePool(data); 4534 goto end; 4535 } 4536 } 4537 4538 if (!no_buf) 4539 ExFreePool(data); 4540 } 4541 4542 KeQuerySystemTime(&time); 4543 win_time_to_unix(time, &now); 4544 4545 if (!pagefile) { 4546 if (fcb->ads) { 4547 if (fileref && fileref->parent) 4548 origii = &fileref->parent->fcb->inode_item; 4549 else { 4550 ERR("no parent fcb found for stream\n"); 4551 Status = STATUS_INTERNAL_ERROR; 4552 goto end; 4553 } 4554 } else 4555 origii = &fcb->inode_item; 4556 4557 origii->transid = Vcb->superblock.generation; 4558 origii->sequence++; 4559 4560 if (!ccb->user_set_change_time) 4561 origii->st_ctime = now; 4562 4563 if (!fcb->ads) { 4564 if (changed_length) { 4565 TRACE("setting st_size to %I64x\n", newlength); 4566 origii->st_size = newlength; 4567 filter |= FILE_NOTIFY_CHANGE_SIZE; 4568 } 4569 4570 fcb->inode_item_changed = true; 4571 } else { 4572 fileref->parent->fcb->inode_item_changed = true; 4573 4574 if (changed_length) 4575 filter |= FILE_NOTIFY_CHANGE_STREAM_SIZE; 4576 4577 filter |= FILE_NOTIFY_CHANGE_STREAM_WRITE; 4578 } 4579 4580 if (!ccb->user_set_write_time) { 4581 origii->st_mtime = now; 4582 filter |= FILE_NOTIFY_CHANGE_LAST_WRITE; 4583 } 4584 4585 mark_fcb_dirty(fcb->ads ? fileref->parent->fcb : fcb); 4586 } 4587 4588 if (changed_length) { 4589 CC_FILE_SIZES ccfs; 4590 4591 ccfs.AllocationSize = fcb->Header.AllocationSize; 4592 ccfs.FileSize = fcb->Header.FileSize; 4593 ccfs.ValidDataLength = fcb->Header.ValidDataLength; 4594 4595 _SEH2_TRY { 4596 CcSetFileSizes(FileObject, &ccfs); 4597 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4598 Status = _SEH2_GetExceptionCode(); 4599 goto end; 4600 } _SEH2_END; 4601 } 4602 4603 fcb->subvol->root_item.ctransid = Vcb->superblock.generation; 4604 fcb->subvol->root_item.ctime = now; 4605 4606 Status = STATUS_SUCCESS; 4607 4608 if (filter != 0) 4609 queue_notification_fcb(fcb->ads ? fileref->parent : fileref, filter, fcb->ads ? FILE_ACTION_MODIFIED_STREAM : FILE_ACTION_MODIFIED, 4610 fcb->ads && fileref->dc ? &fileref->dc->name : NULL); 4611 4612 end: 4613 if (NT_SUCCESS(Status) && FileObject->Flags & FO_SYNCHRONOUS_IO && !paging_io) { 4614 TRACE("CurrentByteOffset was: %I64x\n", FileObject->CurrentByteOffset.QuadPart); 4615 FileObject->CurrentByteOffset.QuadPart = offset.QuadPart + (NT_SUCCESS(Status) ? *length : 0); 4616 TRACE("CurrentByteOffset now: %I64x\n", FileObject->CurrentByteOffset.QuadPart); 4617 } 4618 4619 if (acquired_fcb_lock) 4620 ExReleaseResourceLite(fcb->Header.Resource); 4621 4622 if (acquired_tree_lock) 4623 ExReleaseResourceLite(&Vcb->tree_lock); 4624 4625 if (paging_lock) 4626 ExReleaseResourceLite(fcb->Header.PagingIoResource); 4627 4628 return Status; 4629 } 4630 4631 NTSTATUS write_file(device_extension* Vcb, PIRP Irp, bool wait, bool deferred_write) { 4632 PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp); 4633 void* buf; 4634 NTSTATUS Status; 4635 LARGE_INTEGER offset = IrpSp->Parameters.Write.ByteOffset; 4636 PFILE_OBJECT FileObject = IrpSp->FileObject; 4637 fcb* fcb = FileObject ? FileObject->FsContext : NULL; 4638 LIST_ENTRY rollback; 4639 4640 InitializeListHead(&rollback); 4641 4642 TRACE("write\n"); 4643 4644 Irp->IoStatus.Information = 0; 4645 4646 TRACE("offset = %I64x\n", offset.QuadPart); 4647 TRACE("length = %x\n", IrpSp->Parameters.Write.Length); 4648 4649 if (!Irp->AssociatedIrp.SystemBuffer) { 4650 buf = map_user_buffer(Irp, fcb && fcb->Header.Flags2 & FSRTL_FLAG2_IS_PAGING_FILE ? HighPagePriority : NormalPagePriority); 4651 4652 if (Irp->MdlAddress && !buf) { 4653 ERR("MmGetSystemAddressForMdlSafe returned NULL\n"); 4654 Status = STATUS_INSUFFICIENT_RESOURCES; 4655 goto exit; 4656 } 4657 } else 4658 buf = Irp->AssociatedIrp.SystemBuffer; 4659 4660 TRACE("buf = %p\n", buf); 4661 4662 if (fcb && !(Irp->Flags & IRP_PAGING_IO) && !FsRtlCheckLockForWriteAccess(&fcb->lock, Irp)) { 4663 WARN("tried to write to locked region\n"); 4664 Status = STATUS_FILE_LOCK_CONFLICT; 4665 goto exit; 4666 } 4667 4668 Status = write_file2(Vcb, Irp, offset, buf, &IrpSp->Parameters.Write.Length, Irp->Flags & IRP_PAGING_IO, Irp->Flags & IRP_NOCACHE, 4669 wait, deferred_write, true, &rollback); 4670 4671 if (Status == STATUS_PENDING) 4672 goto exit; 4673 else if (!NT_SUCCESS(Status)) { 4674 ERR("write_file2 returned %08x\n", Status); 4675 goto exit; 4676 } 4677 4678 if (NT_SUCCESS(Status)) { 4679 Irp->IoStatus.Information = IrpSp->Parameters.Write.Length; 4680 4681 if (diskacc && Status != STATUS_PENDING && Irp->Flags & IRP_NOCACHE) { 4682 PETHREAD thread = NULL; 4683 4684 if (Irp->Tail.Overlay.Thread && !IoIsSystemThread(Irp->Tail.Overlay.Thread)) 4685 thread = Irp->Tail.Overlay.Thread; 4686 else if (!IoIsSystemThread(PsGetCurrentThread())) 4687 thread = PsGetCurrentThread(); 4688 else if (IoIsSystemThread(PsGetCurrentThread()) && IoGetTopLevelIrp() == Irp) 4689 thread = PsGetCurrentThread(); 4690 4691 if (thread) 4692 fPsUpdateDiskCounters(PsGetThreadProcess(thread), 0, IrpSp->Parameters.Write.Length, 0, 1, 0); 4693 } 4694 } 4695 4696 exit: 4697 if (NT_SUCCESS(Status)) 4698 clear_rollback(&rollback); 4699 else 4700 do_rollback(Vcb, &rollback); 4701 4702 return Status; 4703 } 4704 4705 _Dispatch_type_(IRP_MJ_WRITE) 4706 _Function_class_(DRIVER_DISPATCH) 4707 NTSTATUS __stdcall drv_write(IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp) { 4708 NTSTATUS Status; 4709 bool top_level; 4710 PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp); 4711 device_extension* Vcb = DeviceObject->DeviceExtension; 4712 PFILE_OBJECT FileObject = IrpSp->FileObject; 4713 fcb* fcb = FileObject ? FileObject->FsContext : NULL; 4714 ccb* ccb = FileObject ? FileObject->FsContext2 : NULL; 4715 bool wait = FileObject ? IoIsOperationSynchronous(Irp) : true; 4716 4717 FsRtlEnterFileSystem(); 4718 4719 top_level = is_top_level(Irp); 4720 4721 if (Vcb && Vcb->type == VCB_TYPE_VOLUME) { 4722 Status = vol_write(DeviceObject, Irp); 4723 goto exit; 4724 } else if (!Vcb || Vcb->type != VCB_TYPE_FS) { 4725 Status = STATUS_INVALID_PARAMETER; 4726 goto end; 4727 } 4728 4729 if (!fcb) { 4730 ERR("fcb was NULL\n"); 4731 Status = STATUS_INVALID_PARAMETER; 4732 goto end; 4733 } 4734 4735 if (!ccb) { 4736 ERR("ccb was NULL\n"); 4737 Status = STATUS_INVALID_PARAMETER; 4738 goto end; 4739 } 4740 4741 if (Irp->RequestorMode == UserMode && !(ccb->access & (FILE_WRITE_DATA | FILE_APPEND_DATA))) { 4742 WARN("insufficient permissions\n"); 4743 Status = STATUS_ACCESS_DENIED; 4744 goto end; 4745 } 4746 4747 if (fcb == Vcb->volume_fcb) { 4748 if (!Vcb->locked || Vcb->locked_fileobj != FileObject) { 4749 ERR("trying to write to volume when not locked, or locked with another FileObject\n"); 4750 Status = STATUS_ACCESS_DENIED; 4751 goto end; 4752 } 4753 4754 TRACE("writing directly to volume\n"); 4755 4756 IoSkipCurrentIrpStackLocation(Irp); 4757 4758 Status = IoCallDriver(Vcb->Vpb->RealDevice, Irp); 4759 goto exit; 4760 } 4761 4762 if (is_subvol_readonly(fcb->subvol, Irp)) { 4763 Status = STATUS_ACCESS_DENIED; 4764 goto end; 4765 } 4766 4767 if (Vcb->readonly) { 4768 Status = STATUS_MEDIA_WRITE_PROTECTED; 4769 goto end; 4770 } 4771 4772 _SEH2_TRY { 4773 if (IrpSp->MinorFunction & IRP_MN_COMPLETE) { 4774 CcMdlWriteComplete(IrpSp->FileObject, &IrpSp->Parameters.Write.ByteOffset, Irp->MdlAddress); 4775 4776 Irp->MdlAddress = NULL; 4777 Status = STATUS_SUCCESS; 4778 } else { 4779 if (!(Irp->Flags & IRP_PAGING_IO)) 4780 FsRtlCheckOplock(fcb_oplock(fcb), Irp, NULL, NULL, NULL); 4781 4782 // Don't offload jobs when doing paging IO - otherwise this can lead to 4783 // deadlocks in CcCopyWrite. 4784 if (Irp->Flags & IRP_PAGING_IO) 4785 wait = true; 4786 4787 Status = write_file(Vcb, Irp, wait, false); 4788 } 4789 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) { 4790 Status = _SEH2_GetExceptionCode(); 4791 } _SEH2_END; 4792 4793 end: 4794 Irp->IoStatus.Status = Status; 4795 4796 TRACE("wrote %u bytes\n", Irp->IoStatus.Information); 4797 4798 if (Status != STATUS_PENDING) 4799 IoCompleteRequest(Irp, IO_NO_INCREMENT); 4800 else { 4801 IoMarkIrpPending(Irp); 4802 4803 if (!add_thread_job(Vcb, Irp)) 4804 Status = do_write_job(Vcb, Irp); 4805 } 4806 4807 exit: 4808 if (top_level) 4809 IoSetTopLevelIrp(NULL); 4810 4811 TRACE("returning %08x\n", Status); 4812 4813 FsRtlExitFileSystem(); 4814 4815 return Status; 4816 } 4817