1<?php 2/** 3 * Base class for all backends using particular storage medium. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program; if not, write to the Free Software Foundation, Inc., 17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 * http://www.gnu.org/copyleft/gpl.html 19 * 20 * @file 21 * @ingroup FileBackend 22 */ 23 24use Wikimedia\AtEase\AtEase; 25use Wikimedia\Timestamp\ConvertibleTimestamp; 26 27/** 28 * @brief Base class for all backends using particular storage medium. 29 * 30 * This class defines the methods as abstract that subclasses must implement. 31 * Outside callers should *not* use functions with "Internal" in the name. 32 * 33 * The FileBackend operations are implemented using basic functions 34 * such as storeInternal(), copyInternal(), deleteInternal() and the like. 35 * This class is also responsible for path resolution and sanitization. 36 * 37 * @stable to extend 38 * @ingroup FileBackend 39 * @since 1.19 40 */ 41abstract class FileBackendStore extends FileBackend { 42 /** @var WANObjectCache */ 43 protected $memCache; 44 /** @var BagOStuff */ 45 protected $srvCache; 46 /** @var MapCacheLRU Map of paths to small (RAM/disk) cache items */ 47 protected $cheapCache; 48 /** @var MapCacheLRU Map of paths to large (RAM/disk) cache items */ 49 protected $expensiveCache; 50 51 /** @var array Map of container names to sharding config */ 52 protected $shardViaHashLevels = []; 53 54 /** @var callable Method to get the MIME type of files */ 55 protected $mimeCallback; 56 57 protected $maxFileSize = 4294967296; // integer bytes (4GiB) 58 59 protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries 60 protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache" 61 protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" 62 63 /** @var false Idiom for "no result due to missing file" (since 1.34) */ 64 protected static $RES_ABSENT = false; 65 /** @var null Idiom for "no result due to I/O errors" (since 1.34) */ 66 protected static $RES_ERROR = null; 67 68 /** @var string File does not exist according to a normal stat query */ 69 protected static $ABSENT_NORMAL = 'FNE-N'; 70 /** @var string File does not exist according to a "latest"-mode stat query */ 71 protected static $ABSENT_LATEST = 'FNE-L'; 72 73 /** 74 * @see FileBackend::__construct() 75 * Additional $config params include: 76 * - srvCache : BagOStuff cache to APC or the like. 77 * - wanCache : WANObjectCache object to use for persistent caching. 78 * - mimeCallback : Callback that takes (storage path, content, file system path) and 79 * returns the MIME type of the file or 'unknown/unknown'. The file 80 * system path parameter should be used if the content one is null. 81 * 82 * @stable to call 83 * 84 * @param array $config 85 */ 86 public function __construct( array $config ) { 87 parent::__construct( $config ); 88 $this->mimeCallback = $config['mimeCallback'] ?? null; 89 $this->srvCache = new EmptyBagOStuff(); // disabled by default 90 $this->memCache = WANObjectCache::newEmpty(); // disabled by default 91 $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE ); 92 $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE ); 93 } 94 95 /** 96 * Get the maximum allowable file size given backend 97 * medium restrictions and basic performance constraints. 98 * Do not call this function from places outside FileBackend and FileOp. 99 * 100 * @return int Bytes 101 */ 102 final public function maxFileSizeInternal() { 103 return $this->maxFileSize; 104 } 105 106 /** 107 * Check if a file can be created or changed at a given storage path in the backend 108 * 109 * FS backends should check that the parent directory exists, files can be written 110 * under it, and that any file already there is both readable and writable. 111 * Backends using key/value stores should check if the container exists. 112 * 113 * @param string $storagePath 114 * @return bool 115 */ 116 abstract public function isPathUsableInternal( $storagePath ); 117 118 /** 119 * Create a file in the backend with the given contents. 120 * This will overwrite any file that exists at the destination. 121 * Do not call this function from places outside FileBackend and FileOp. 122 * 123 * $params include: 124 * - content : the raw file contents 125 * - dst : destination storage path 126 * - headers : HTTP header name/value map 127 * - async : StatusValue will be returned immediately if supported. 128 * If the StatusValue is OK, then its value field will be 129 * set to a FileBackendStoreOpHandle object. 130 * - dstExists : Whether a file exists at the destination (optimization). 131 * Callers can use "false" if no existing file is being changed. 132 * 133 * @param array $params 134 * @return StatusValue 135 */ 136 final public function createInternal( array $params ) { 137 /** @noinspection PhpUnusedLocalVariableInspection */ 138 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 139 140 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { 141 $status = $this->newStatus( 'backend-fail-maxsize', 142 $params['dst'], $this->maxFileSizeInternal() ); 143 } else { 144 $status = $this->doCreateInternal( $params ); 145 $this->clearCache( [ $params['dst'] ] ); 146 if ( $params['dstExists'] ?? true ) { 147 $this->deleteFileCache( $params['dst'] ); // persistent cache 148 } 149 } 150 151 return $status; 152 } 153 154 /** 155 * @see FileBackendStore::createInternal() 156 * @param array $params 157 * @return StatusValue 158 */ 159 abstract protected function doCreateInternal( array $params ); 160 161 /** 162 * Store a file into the backend from a file on disk. 163 * This will overwrite any file that exists at the destination. 164 * Do not call this function from places outside FileBackend and FileOp. 165 * 166 * $params include: 167 * - src : source path on disk 168 * - dst : destination storage path 169 * - headers : HTTP header name/value map 170 * - async : StatusValue will be returned immediately if supported. 171 * If the StatusValue is OK, then its value field will be 172 * set to a FileBackendStoreOpHandle object. 173 * - dstExists : Whether a file exists at the destination (optimization). 174 * Callers can use "false" if no existing file is being changed. 175 * 176 * @param array $params 177 * @return StatusValue 178 */ 179 final public function storeInternal( array $params ) { 180 /** @noinspection PhpUnusedLocalVariableInspection */ 181 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 182 183 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { 184 $status = $this->newStatus( 'backend-fail-maxsize', 185 $params['dst'], $this->maxFileSizeInternal() ); 186 } else { 187 $status = $this->doStoreInternal( $params ); 188 $this->clearCache( [ $params['dst'] ] ); 189 if ( $params['dstExists'] ?? true ) { 190 $this->deleteFileCache( $params['dst'] ); // persistent cache 191 } 192 } 193 194 return $status; 195 } 196 197 /** 198 * @see FileBackendStore::storeInternal() 199 * @param array $params 200 * @return StatusValue 201 */ 202 abstract protected function doStoreInternal( array $params ); 203 204 /** 205 * Copy a file from one storage path to another in the backend. 206 * This will overwrite any file that exists at the destination. 207 * Do not call this function from places outside FileBackend and FileOp. 208 * 209 * $params include: 210 * - src : source storage path 211 * - dst : destination storage path 212 * - ignoreMissingSource : do nothing if the source file does not exist 213 * - headers : HTTP header name/value map 214 * - async : StatusValue will be returned immediately if supported. 215 * If the StatusValue is OK, then its value field will be 216 * set to a FileBackendStoreOpHandle object. 217 * - dstExists : Whether a file exists at the destination (optimization). 218 * Callers can use "false" if no existing file is being changed. 219 * 220 * @param array $params 221 * @return StatusValue 222 */ 223 final public function copyInternal( array $params ) { 224 /** @noinspection PhpUnusedLocalVariableInspection */ 225 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 226 227 $status = $this->doCopyInternal( $params ); 228 $this->clearCache( [ $params['dst'] ] ); 229 if ( $params['dstExists'] ?? true ) { 230 $this->deleteFileCache( $params['dst'] ); // persistent cache 231 } 232 233 return $status; 234 } 235 236 /** 237 * @see FileBackendStore::copyInternal() 238 * @param array $params 239 * @return StatusValue 240 */ 241 abstract protected function doCopyInternal( array $params ); 242 243 /** 244 * Delete a file at the storage path. 245 * Do not call this function from places outside FileBackend and FileOp. 246 * 247 * $params include: 248 * - src : source storage path 249 * - ignoreMissingSource : do nothing if the source file does not exist 250 * - async : StatusValue will be returned immediately if supported. 251 * If the StatusValue is OK, then its value field will be 252 * set to a FileBackendStoreOpHandle object. 253 * 254 * @param array $params 255 * @return StatusValue 256 */ 257 final public function deleteInternal( array $params ) { 258 /** @noinspection PhpUnusedLocalVariableInspection */ 259 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 260 261 $status = $this->doDeleteInternal( $params ); 262 $this->clearCache( [ $params['src'] ] ); 263 $this->deleteFileCache( $params['src'] ); // persistent cache 264 return $status; 265 } 266 267 /** 268 * @see FileBackendStore::deleteInternal() 269 * @param array $params 270 * @return StatusValue 271 */ 272 abstract protected function doDeleteInternal( array $params ); 273 274 /** 275 * Move a file from one storage path to another in the backend. 276 * This will overwrite any file that exists at the destination. 277 * Do not call this function from places outside FileBackend and FileOp. 278 * 279 * $params include: 280 * - src : source storage path 281 * - dst : destination storage path 282 * - ignoreMissingSource : do nothing if the source file does not exist 283 * - headers : HTTP header name/value map 284 * - async : StatusValue will be returned immediately if supported. 285 * If the StatusValue is OK, then its value field will be 286 * set to a FileBackendStoreOpHandle object. 287 * - dstExists : Whether a file exists at the destination (optimization). 288 * Callers can use "false" if no existing file is being changed. 289 * 290 * @param array $params 291 * @return StatusValue 292 */ 293 final public function moveInternal( array $params ) { 294 /** @noinspection PhpUnusedLocalVariableInspection */ 295 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 296 297 $status = $this->doMoveInternal( $params ); 298 $this->clearCache( [ $params['src'], $params['dst'] ] ); 299 $this->deleteFileCache( $params['src'] ); // persistent cache 300 if ( $params['dstExists'] ?? true ) { 301 $this->deleteFileCache( $params['dst'] ); // persistent cache 302 } 303 304 return $status; 305 } 306 307 /** 308 * @see FileBackendStore::moveInternal() 309 * @stable to override 310 * @param array $params 311 * @return StatusValue 312 */ 313 protected function doMoveInternal( array $params ) { 314 unset( $params['async'] ); // two steps, won't work here :) 315 $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); 316 $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); 317 // Copy source to dest 318 $status = $this->copyInternal( $params ); 319 if ( $nsrc !== $ndst && $status->isOK() ) { 320 // Delete source (only fails due to races or network problems) 321 $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) ); 322 $status->setResult( true, $status->value ); // ignore delete() errors 323 } 324 325 return $status; 326 } 327 328 /** 329 * Alter metadata for a file at the storage path. 330 * Do not call this function from places outside FileBackend and FileOp. 331 * 332 * $params include: 333 * - src : source storage path 334 * - headers : HTTP header name/value map 335 * - async : StatusValue will be returned immediately if supported. 336 * If the StatusValue is OK, then its value field will be 337 * set to a FileBackendStoreOpHandle object. 338 * 339 * @param array $params 340 * @return StatusValue 341 */ 342 final public function describeInternal( array $params ) { 343 /** @noinspection PhpUnusedLocalVariableInspection */ 344 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 345 346 if ( count( $params['headers'] ) ) { 347 $status = $this->doDescribeInternal( $params ); 348 $this->clearCache( [ $params['src'] ] ); 349 $this->deleteFileCache( $params['src'] ); // persistent cache 350 } else { 351 $status = $this->newStatus(); // nothing to do 352 } 353 354 return $status; 355 } 356 357 /** 358 * @see FileBackendStore::describeInternal() 359 * @stable to override 360 * @param array $params 361 * @return StatusValue 362 */ 363 protected function doDescribeInternal( array $params ) { 364 return $this->newStatus(); 365 } 366 367 /** 368 * No-op file operation that does nothing. 369 * Do not call this function from places outside FileBackend and FileOp. 370 * 371 * @param array $params 372 * @return StatusValue 373 */ 374 final public function nullInternal( array $params ) { 375 return $this->newStatus(); 376 } 377 378 final public function concatenate( array $params ) { 379 /** @noinspection PhpUnusedLocalVariableInspection */ 380 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 381 $status = $this->newStatus(); 382 383 // Try to lock the source files for the scope of this function 384 /** @noinspection PhpUnusedLocalVariableInspection */ 385 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); 386 if ( $status->isOK() ) { 387 // Actually do the file concatenation... 388 $start_time = microtime( true ); 389 $status->merge( $this->doConcatenate( $params ) ); 390 $sec = microtime( true ) - $start_time; 391 if ( !$status->isOK() ) { 392 $this->logger->error( static::class . "-{$this->name}" . 393 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" ); 394 } 395 } 396 397 return $status; 398 } 399 400 /** 401 * @see FileBackendStore::concatenate() 402 * @stable to override 403 * @param array $params 404 * @return StatusValue 405 */ 406 protected function doConcatenate( array $params ) { 407 $status = $this->newStatus(); 408 $tmpPath = $params['dst']; // convenience 409 unset( $params['latest'] ); // sanity 410 411 // Check that the specified temp file is valid... 412 AtEase::suppressWarnings(); 413 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 ); 414 AtEase::restoreWarnings(); 415 if ( !$ok ) { // not present or not empty 416 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 417 418 return $status; 419 } 420 421 // Get local FS versions of the chunks needed for the concatenation... 422 $fsFiles = $this->getLocalReferenceMulti( $params ); 423 foreach ( $fsFiles as $path => &$fsFile ) { 424 if ( !$fsFile ) { // chunk failed to download? 425 $fsFile = $this->getLocalReference( [ 'src' => $path ] ); 426 if ( !$fsFile ) { // retry failed? 427 $status->fatal( 'backend-fail-read', $path ); 428 429 return $status; 430 } 431 } 432 } 433 unset( $fsFile ); // unset reference so we can reuse $fsFile 434 435 // Get a handle for the destination temp file 436 $tmpHandle = fopen( $tmpPath, 'ab' ); 437 if ( $tmpHandle === false ) { 438 $status->fatal( 'backend-fail-opentemp', $tmpPath ); 439 440 return $status; 441 } 442 443 // Build up the temp file using the source chunks (in order)... 444 foreach ( $fsFiles as $virtualSource => $fsFile ) { 445 // Get a handle to the local FS version 446 $sourceHandle = fopen( $fsFile->getPath(), 'rb' ); 447 if ( $sourceHandle === false ) { 448 fclose( $tmpHandle ); 449 $status->fatal( 'backend-fail-read', $virtualSource ); 450 451 return $status; 452 } 453 // Append chunk to file (pass chunk size to avoid magic quotes) 454 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { 455 fclose( $sourceHandle ); 456 fclose( $tmpHandle ); 457 $status->fatal( 'backend-fail-writetemp', $tmpPath ); 458 459 return $status; 460 } 461 fclose( $sourceHandle ); 462 } 463 if ( !fclose( $tmpHandle ) ) { 464 $status->fatal( 'backend-fail-closetemp', $tmpPath ); 465 466 return $status; 467 } 468 469 clearstatcache(); // temp file changed 470 471 return $status; 472 } 473 474 final protected function doPrepare( array $params ) { 475 /** @noinspection PhpUnusedLocalVariableInspection */ 476 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 477 $status = $this->newStatus(); 478 479 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 480 if ( $dir === null ) { 481 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 482 483 return $status; // invalid storage path 484 } 485 486 if ( $shard !== null ) { // confined to a single container/shard 487 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); 488 } else { // directory is on several shards 489 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 490 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 491 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 492 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 493 } 494 } 495 496 return $status; 497 } 498 499 /** 500 * @see FileBackendStore::doPrepare() 501 * @stable to override 502 * @param string $container 503 * @param string $dir 504 * @param array $params 505 * @return StatusValue 506 */ 507 protected function doPrepareInternal( $container, $dir, array $params ) { 508 return $this->newStatus(); 509 } 510 511 final protected function doSecure( array $params ) { 512 /** @noinspection PhpUnusedLocalVariableInspection */ 513 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 514 $status = $this->newStatus(); 515 516 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 517 if ( $dir === null ) { 518 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 519 520 return $status; // invalid storage path 521 } 522 523 if ( $shard !== null ) { // confined to a single container/shard 524 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); 525 } else { // directory is on several shards 526 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 527 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 528 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 529 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 530 } 531 } 532 533 return $status; 534 } 535 536 /** 537 * @see FileBackendStore::doSecure() 538 * @stable to override 539 * @param string $container 540 * @param string $dir 541 * @param array $params 542 * @return StatusValue 543 */ 544 protected function doSecureInternal( $container, $dir, array $params ) { 545 return $this->newStatus(); 546 } 547 548 final protected function doPublish( array $params ) { 549 /** @noinspection PhpUnusedLocalVariableInspection */ 550 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 551 $status = $this->newStatus(); 552 553 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 554 if ( $dir === null ) { 555 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 556 557 return $status; // invalid storage path 558 } 559 560 if ( $shard !== null ) { // confined to a single container/shard 561 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); 562 } else { // directory is on several shards 563 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 564 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 565 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 566 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 567 } 568 } 569 570 return $status; 571 } 572 573 /** 574 * @see FileBackendStore::doPublish() 575 * @stable to override 576 * @param string $container 577 * @param string $dir 578 * @param array $params 579 * @return StatusValue 580 */ 581 protected function doPublishInternal( $container, $dir, array $params ) { 582 return $this->newStatus(); 583 } 584 585 final protected function doClean( array $params ) { 586 /** @noinspection PhpUnusedLocalVariableInspection */ 587 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 588 $status = $this->newStatus(); 589 590 // Recursive: first delete all empty subdirs recursively 591 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) { 592 $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] ); 593 if ( $subDirsRel !== null ) { // no errors 594 foreach ( $subDirsRel as $subDirRel ) { 595 $subDir = $params['dir'] . "/{$subDirRel}"; // full path 596 $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) ); 597 } 598 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends) 599 } 600 } 601 602 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 603 if ( $dir === null ) { 604 $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); 605 606 return $status; // invalid storage path 607 } 608 609 // Attempt to lock this directory... 610 $filesLockEx = [ $params['dir'] ]; 611 /** @noinspection PhpUnusedLocalVariableInspection */ 612 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); 613 if ( !$status->isOK() ) { 614 return $status; // abort 615 } 616 617 if ( $shard !== null ) { // confined to a single container/shard 618 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); 619 $this->deleteContainerCache( $fullCont ); // purge cache 620 } else { // directory is on several shards 621 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 622 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 623 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 624 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); 625 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache 626 } 627 } 628 629 return $status; 630 } 631 632 /** 633 * @see FileBackendStore::doClean() 634 * @stable to override 635 * @param string $container 636 * @param string $dir 637 * @param array $params 638 * @return StatusValue 639 */ 640 protected function doCleanInternal( $container, $dir, array $params ) { 641 return $this->newStatus(); 642 } 643 644 final public function fileExists( array $params ) { 645 /** @noinspection PhpUnusedLocalVariableInspection */ 646 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 647 648 $stat = $this->getFileStat( $params ); 649 if ( is_array( $stat ) ) { 650 return true; 651 } 652 653 return ( $stat === self::$RES_ABSENT ) ? false : self::EXISTENCE_ERROR; 654 } 655 656 final public function getFileTimestamp( array $params ) { 657 /** @noinspection PhpUnusedLocalVariableInspection */ 658 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 659 660 $stat = $this->getFileStat( $params ); 661 if ( is_array( $stat ) ) { 662 return $stat['mtime']; 663 } 664 665 return self::TIMESTAMP_FAIL; // all failure cases 666 } 667 668 final public function getFileSize( array $params ) { 669 /** @noinspection PhpUnusedLocalVariableInspection */ 670 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 671 672 $stat = $this->getFileStat( $params ); 673 if ( is_array( $stat ) ) { 674 return $stat['size']; 675 } 676 677 return self::SIZE_FAIL; // all failure cases 678 } 679 680 final public function getFileStat( array $params ) { 681 /** @noinspection PhpUnusedLocalVariableInspection */ 682 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 683 684 $path = self::normalizeStoragePath( $params['src'] ); 685 if ( $path === null ) { 686 return self::STAT_ERROR; // invalid storage path 687 } 688 689 // Whether to bypass cache except for process cache entries loaded directly from 690 // high consistency backend queries (caller handles any cache flushing and locking) 691 $latest = !empty( $params['latest'] ); 692 // Whether to ignore cache entries missing the SHA-1 field for existing files 693 $requireSHA1 = !empty( $params['requireSHA1'] ); 694 695 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL ); 696 // Load the persistent stat cache into process cache if needed 697 if ( !$latest ) { 698 if ( 699 // File stat is not in process cache 700 $stat === null || 701 // Key/value store backends might opportunistically set file stat process 702 // cache entries from object listings that do not include the SHA-1. In that 703 // case, loading the persistent stat cache will likely yield the SHA-1. 704 ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) ) 705 ) { 706 $this->primeFileCache( [ $path ] ); 707 // Get any newly process-cached entry 708 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL ); 709 } 710 } 711 712 if ( is_array( $stat ) ) { 713 if ( 714 ( !$latest || $stat['latest'] ) && 715 ( !$requireSHA1 || isset( $stat['sha1'] ) ) 716 ) { 717 return $stat; 718 } 719 } elseif ( $stat === self::$ABSENT_LATEST ) { 720 return self::STAT_ABSENT; 721 } elseif ( $stat === self::$ABSENT_NORMAL ) { 722 if ( !$latest ) { 723 return self::STAT_ABSENT; 724 } 725 } 726 727 // Load the file stat from the backend and update caches 728 $stat = $this->doGetFileStat( $params ); 729 $this->ingestFreshFileStats( [ $path => $stat ], $latest ); 730 731 if ( is_array( $stat ) ) { 732 return $stat; 733 } 734 735 return ( $stat === self::$RES_ERROR ) ? self::STAT_ERROR : self::STAT_ABSENT; 736 } 737 738 /** 739 * Ingest file stat entries that just came from querying the backend (not cache) 740 * 741 * @param array[]|bool[]|null[] $stats Map of (path => doGetFileStat() stype result) 742 * @param bool $latest Whether doGetFileStat()/doGetFileStatMulti() had the 'latest' flag 743 * @return bool Whether all files have non-error stat replies 744 */ 745 final protected function ingestFreshFileStats( array $stats, $latest ) { 746 $success = true; 747 748 foreach ( $stats as $path => $stat ) { 749 if ( is_array( $stat ) ) { 750 // Strongly consistent backends might automatically set this flag 751 $stat['latest'] = $stat['latest'] ?? $latest; 752 753 $this->cheapCache->setField( $path, 'stat', $stat ); 754 if ( isset( $stat['sha1'] ) ) { 755 // Some backends store the SHA-1 hash as metadata 756 $this->cheapCache->setField( 757 $path, 758 'sha1', 759 [ 'hash' => $stat['sha1'], 'latest' => $latest ] 760 ); 761 } 762 if ( isset( $stat['xattr'] ) ) { 763 // Some backends store custom headers/metadata 764 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); 765 $this->cheapCache->setField( 766 $path, 767 'xattr', 768 [ 'map' => $stat['xattr'], 'latest' => $latest ] 769 ); 770 } 771 // Update persistent cache (@TODO: set all entries in one batch) 772 $this->setFileCache( $path, $stat ); 773 } elseif ( $stat === self::$RES_ABSENT ) { 774 $this->cheapCache->setField( 775 $path, 776 'stat', 777 $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL 778 ); 779 $this->cheapCache->setField( 780 $path, 781 'xattr', 782 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ] 783 ); 784 $this->cheapCache->setField( 785 $path, 786 'sha1', 787 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ] 788 ); 789 $this->logger->debug( 790 __METHOD__ . ': File {path} does not exist', 791 [ 'path' => $path ] 792 ); 793 } else { 794 $success = false; 795 $this->logger->error( 796 __METHOD__ . ': Could not stat file {path}', 797 [ 'path' => $path ] 798 ); 799 } 800 } 801 802 return $success; 803 } 804 805 /** 806 * @see FileBackendStore::getFileStat() 807 * @param array $params 808 */ 809 abstract protected function doGetFileStat( array $params ); 810 811 public function getFileContentsMulti( array $params ) { 812 /** @noinspection PhpUnusedLocalVariableInspection */ 813 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 814 815 $params = $this->setConcurrencyFlags( $params ); 816 $contents = $this->doGetFileContentsMulti( $params ); 817 foreach ( $contents as $path => $content ) { 818 if ( !is_string( $content ) ) { 819 $contents[$path] = self::CONTENT_FAIL; // used for all failure cases 820 } 821 } 822 823 return $contents; 824 } 825 826 /** 827 * @see FileBackendStore::getFileContentsMulti() 828 * @stable to override 829 * @param array $params 830 * @return string[]|bool[]|null[] Map of (path => string, false (missing), or null (error)) 831 */ 832 protected function doGetFileContentsMulti( array $params ) { 833 $contents = []; 834 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 835 if ( $fsFile instanceof FSFile ) { 836 AtEase::suppressWarnings(); 837 $content = file_get_contents( $fsFile->getPath() ); 838 AtEase::restoreWarnings(); 839 $contents[$path] = is_string( $content ) ? $content : self::$RES_ERROR; 840 } elseif ( $fsFile === self::$RES_ABSENT ) { 841 $contents[$path] = self::$RES_ABSENT; 842 } else { 843 $contents[$path] = self::$RES_ERROR; 844 } 845 } 846 847 return $contents; 848 } 849 850 final public function getFileXAttributes( array $params ) { 851 /** @noinspection PhpUnusedLocalVariableInspection */ 852 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 853 854 $path = self::normalizeStoragePath( $params['src'] ); 855 if ( $path === null ) { 856 return self::XATTRS_FAIL; // invalid storage path 857 } 858 $latest = !empty( $params['latest'] ); // use latest data? 859 if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) { 860 $stat = $this->cheapCache->getField( $path, 'xattr' ); 861 // If we want the latest data, check that this cached 862 // value was in fact fetched with the latest available data. 863 if ( !$latest || $stat['latest'] ) { 864 return $stat['map']; 865 } 866 } 867 $fields = $this->doGetFileXAttributes( $params ); 868 if ( is_array( $fields ) ) { 869 $fields = self::normalizeXAttributes( $fields ); 870 $this->cheapCache->setField( 871 $path, 872 'xattr', 873 [ 'map' => $fields, 'latest' => $latest ] 874 ); 875 } elseif ( $fields === self::$RES_ABSENT ) { 876 $this->cheapCache->setField( 877 $path, 878 'xattr', 879 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ] 880 ); 881 } else { 882 $fields = self::XATTRS_FAIL; // used for all failure cases 883 } 884 885 return $fields; 886 } 887 888 /** 889 * @see FileBackendStore::getFileXAttributes() 890 * @stable to override 891 * @param array $params 892 * @return array[][]|false|null Attributes, false (missing file), or null (error) 893 */ 894 protected function doGetFileXAttributes( array $params ) { 895 return [ 'headers' => [], 'metadata' => [] ]; // not supported 896 } 897 898 final public function getFileSha1Base36( array $params ) { 899 /** @noinspection PhpUnusedLocalVariableInspection */ 900 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 901 902 $path = self::normalizeStoragePath( $params['src'] ); 903 if ( $path === null ) { 904 return self::SHA1_FAIL; // invalid storage path 905 } 906 $latest = !empty( $params['latest'] ); // use latest data? 907 if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) { 908 $stat = $this->cheapCache->getField( $path, 'sha1' ); 909 // If we want the latest data, check that this cached 910 // value was in fact fetched with the latest available data. 911 if ( !$latest || $stat['latest'] ) { 912 return $stat['hash']; 913 } 914 } 915 $sha1 = $this->doGetFileSha1Base36( $params ); 916 if ( is_string( $sha1 ) ) { 917 $this->cheapCache->setField( 918 $path, 919 'sha1', 920 [ 'hash' => $sha1, 'latest' => $latest ] 921 ); 922 } elseif ( $sha1 === self::$RES_ABSENT ) { 923 $this->cheapCache->setField( 924 $path, 925 'sha1', 926 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ] 927 ); 928 } else { 929 $sha1 = self::SHA1_FAIL; // used for all failure cases 930 } 931 932 return $sha1; 933 } 934 935 /** 936 * @see FileBackendStore::getFileSha1Base36() 937 * @stable to override 938 * @param array $params 939 * @return bool|string|null SHA1, false (missing file), or null (error) 940 */ 941 protected function doGetFileSha1Base36( array $params ) { 942 $fsFile = $this->getLocalReference( $params ); 943 if ( $fsFile instanceof FSFile ) { 944 $sha1 = $fsFile->getSha1Base36(); 945 946 return is_string( $sha1 ) ? $sha1 : self::$RES_ERROR; 947 } 948 949 return ( $fsFile === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT; 950 } 951 952 final public function getFileProps( array $params ) { 953 /** @noinspection PhpUnusedLocalVariableInspection */ 954 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 955 956 $fsFile = $this->getLocalReference( $params ); 957 958 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); 959 } 960 961 final public function getLocalReferenceMulti( array $params ) { 962 /** @noinspection PhpUnusedLocalVariableInspection */ 963 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 964 965 $params = $this->setConcurrencyFlags( $params ); 966 967 $fsFiles = []; // (path => FSFile) 968 $latest = !empty( $params['latest'] ); // use latest data? 969 // Reuse any files already in process cache... 970 foreach ( $params['srcs'] as $src ) { 971 $path = self::normalizeStoragePath( $src ); 972 if ( $path === null ) { 973 $fsFiles[$src] = null; // invalid storage path 974 } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) { 975 $val = $this->expensiveCache->getField( $path, 'localRef' ); 976 // If we want the latest data, check that this cached 977 // value was in fact fetched with the latest available data. 978 if ( !$latest || $val['latest'] ) { 979 $fsFiles[$src] = $val['object']; 980 } 981 } 982 } 983 // Fetch local references of any remaning files... 984 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) ); 985 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { 986 if ( $fsFile instanceof FSFile ) { 987 $fsFiles[$path] = $fsFile; 988 $this->expensiveCache->setField( 989 $path, 990 'localRef', 991 [ 'object' => $fsFile, 'latest' => $latest ] 992 ); 993 } else { 994 $fsFiles[$path] = null; // used for all failure cases 995 } 996 } 997 998 return $fsFiles; 999 } 1000 1001 /** 1002 * @see FileBackendStore::getLocalReferenceMulti() 1003 * @stable to override 1004 * @param array $params 1005 * @return string[]|bool[]|null[] Map of (path => FSFile, false (missing), or null (error)) 1006 */ 1007 protected function doGetLocalReferenceMulti( array $params ) { 1008 return $this->doGetLocalCopyMulti( $params ); 1009 } 1010 1011 final public function getLocalCopyMulti( array $params ) { 1012 /** @noinspection PhpUnusedLocalVariableInspection */ 1013 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1014 1015 $params = $this->setConcurrencyFlags( $params ); 1016 $tmpFiles = $this->doGetLocalCopyMulti( $params ); 1017 foreach ( $tmpFiles as $path => $tmpFile ) { 1018 if ( !$tmpFile ) { 1019 $tmpFiles[$path] = null; // used for all failure cases 1020 } 1021 } 1022 1023 return $tmpFiles; 1024 } 1025 1026 /** 1027 * @see FileBackendStore::getLocalCopyMulti() 1028 * @param array $params 1029 * @return string[]|bool[]|null[] Map of (path => TempFSFile, false (missing), or null (error)) 1030 */ 1031 abstract protected function doGetLocalCopyMulti( array $params ); 1032 1033 /** 1034 * @see FileBackend::getFileHttpUrl() 1035 * @stable to override 1036 * @param array $params 1037 * @return string|null 1038 */ 1039 public function getFileHttpUrl( array $params ) { 1040 return self::TEMPURL_ERROR; // not supported 1041 } 1042 1043 final public function streamFile( array $params ) { 1044 /** @noinspection PhpUnusedLocalVariableInspection */ 1045 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1046 $status = $this->newStatus(); 1047 1048 // Always set some fields for subclass convenience 1049 $params['options'] = $params['options'] ?? []; 1050 $params['headers'] = $params['headers'] ?? []; 1051 1052 // Don't stream it out as text/html if there was a PHP error 1053 if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) { 1054 print "Headers already sent, terminating.\n"; 1055 $status->fatal( 'backend-fail-stream', $params['src'] ); 1056 return $status; 1057 } 1058 1059 $status->merge( $this->doStreamFile( $params ) ); 1060 1061 return $status; 1062 } 1063 1064 /** 1065 * @see FileBackendStore::streamFile() 1066 * @stable to override 1067 * @param array $params 1068 * @return StatusValue 1069 */ 1070 protected function doStreamFile( array $params ) { 1071 $status = $this->newStatus(); 1072 1073 $flags = 0; 1074 $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0; 1075 $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0; 1076 1077 $fsFile = $this->getLocalReference( $params ); 1078 if ( $fsFile ) { 1079 $streamer = new HTTPFileStreamer( 1080 $fsFile->getPath(), 1081 [ 1082 'obResetFunc' => $this->obResetFunc, 1083 'streamMimeFunc' => $this->streamMimeFunc 1084 ] 1085 ); 1086 $res = $streamer->stream( $params['headers'], true, $params['options'], $flags ); 1087 } else { 1088 $res = false; 1089 HTTPFileStreamer::send404Message( $params['src'], $flags ); 1090 } 1091 1092 if ( !$res ) { 1093 $status->fatal( 'backend-fail-stream', $params['src'] ); 1094 } 1095 1096 return $status; 1097 } 1098 1099 final public function directoryExists( array $params ) { 1100 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 1101 if ( $dir === null ) { 1102 return self::EXISTENCE_ERROR; // invalid storage path 1103 } 1104 if ( $shard !== null ) { // confined to a single container/shard 1105 return $this->doDirectoryExists( $fullCont, $dir, $params ); 1106 } else { // directory is on several shards 1107 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 1108 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 1109 $res = false; // response 1110 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { 1111 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params ); 1112 if ( $exists === true ) { 1113 $res = true; 1114 break; // found one! 1115 } elseif ( $exists === self::$RES_ERROR ) { 1116 $res = self::EXISTENCE_ERROR; 1117 } 1118 } 1119 1120 return $res; 1121 } 1122 } 1123 1124 /** 1125 * @see FileBackendStore::directoryExists() 1126 * 1127 * @param string $container Resolved container name 1128 * @param string $dir Resolved path relative to container 1129 * @param array $params 1130 * @return bool|null 1131 */ 1132 abstract protected function doDirectoryExists( $container, $dir, array $params ); 1133 1134 final public function getDirectoryList( array $params ) { 1135 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 1136 if ( $dir === null ) { 1137 return self::EXISTENCE_ERROR; // invalid storage path 1138 } 1139 if ( $shard !== null ) { 1140 // File listing is confined to a single container/shard 1141 return $this->getDirectoryListInternal( $fullCont, $dir, $params ); 1142 } else { 1143 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 1144 // File listing spans multiple containers/shards 1145 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 1146 1147 return new FileBackendStoreShardDirIterator( $this, 1148 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 1149 } 1150 } 1151 1152 /** 1153 * Do not call this function from places outside FileBackend 1154 * 1155 * @see FileBackendStore::getDirectoryList() 1156 * 1157 * @param string $container Resolved container name 1158 * @param string $dir Resolved path relative to container 1159 * @param array $params 1160 * @return Traversable|array|null Iterable list or null (error) 1161 */ 1162 abstract public function getDirectoryListInternal( $container, $dir, array $params ); 1163 1164 final public function getFileList( array $params ) { 1165 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); 1166 if ( $dir === null ) { 1167 return self::LIST_ERROR; // invalid storage path 1168 } 1169 if ( $shard !== null ) { 1170 // File listing is confined to a single container/shard 1171 return $this->getFileListInternal( $fullCont, $dir, $params ); 1172 } else { 1173 $this->logger->debug( __METHOD__ . ": iterating over all container shards." ); 1174 // File listing spans multiple containers/shards 1175 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); 1176 1177 return new FileBackendStoreShardFileIterator( $this, 1178 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); 1179 } 1180 } 1181 1182 /** 1183 * Do not call this function from places outside FileBackend 1184 * 1185 * @see FileBackendStore::getFileList() 1186 * 1187 * @param string $container Resolved container name 1188 * @param string $dir Resolved path relative to container 1189 * @param array $params 1190 * @return Traversable|string[]|null Iterable list or null (error) 1191 */ 1192 abstract public function getFileListInternal( $container, $dir, array $params ); 1193 1194 /** 1195 * Return a list of FileOp objects from a list of operations. 1196 * Do not call this function from places outside FileBackend. 1197 * 1198 * The result must have the same number of items as the input. 1199 * An exception is thrown if an unsupported operation is requested. 1200 * 1201 * @param array[] $ops Same format as doOperations() 1202 * @return FileOp[] 1203 * @throws FileBackendError 1204 */ 1205 final public function getOperationsInternal( array $ops ) { 1206 $supportedOps = [ 1207 'store' => StoreFileOp::class, 1208 'copy' => CopyFileOp::class, 1209 'move' => MoveFileOp::class, 1210 'delete' => DeleteFileOp::class, 1211 'create' => CreateFileOp::class, 1212 'describe' => DescribeFileOp::class, 1213 'null' => NullFileOp::class 1214 ]; 1215 1216 $performOps = []; // array of FileOp objects 1217 // Build up ordered array of FileOps... 1218 foreach ( $ops as $operation ) { 1219 $opName = $operation['op']; 1220 if ( isset( $supportedOps[$opName] ) ) { 1221 $class = $supportedOps[$opName]; 1222 // Get params for this operation 1223 $params = $operation; 1224 // Append the FileOp class 1225 $performOps[] = new $class( $this, $params, $this->logger ); 1226 } else { 1227 throw new FileBackendError( "Operation '$opName' is not supported." ); 1228 } 1229 } 1230 1231 return $performOps; 1232 } 1233 1234 /** 1235 * Get a list of storage paths to lock for a list of operations 1236 * Returns an array with LockManager::LOCK_UW (shared locks) and 1237 * LockManager::LOCK_EX (exclusive locks) keys, each corresponding 1238 * to a list of storage paths to be locked. All returned paths are 1239 * normalized. 1240 * 1241 * @param FileOp[] $performOps List of FileOp objects 1242 * @return string[][] (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) 1243 */ 1244 final public function getPathsToLockForOpsInternal( array $performOps ) { 1245 // Build up a list of files to lock... 1246 $paths = [ 'sh' => [], 'ex' => [] ]; 1247 foreach ( $performOps as $fileOp ) { 1248 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() ); 1249 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() ); 1250 } 1251 // Optimization: if doing an EX lock anyway, don't also set an SH one 1252 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] ); 1253 // Get a shared lock on the parent directory of each path changed 1254 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); 1255 1256 return [ 1257 LockManager::LOCK_UW => $paths['sh'], 1258 LockManager::LOCK_EX => $paths['ex'] 1259 ]; 1260 } 1261 1262 public function getScopedLocksForOps( array $ops, StatusValue $status ) { 1263 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); 1264 1265 return $this->getScopedFileLocks( $paths, 'mixed', $status ); 1266 } 1267 1268 final protected function doOperationsInternal( array $ops, array $opts ) { 1269 /** @noinspection PhpUnusedLocalVariableInspection */ 1270 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1271 $status = $this->newStatus(); 1272 1273 // Fix up custom header name/value pairs 1274 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops ); 1275 // Build up a list of FileOps and involved paths 1276 $fileOps = $this->getOperationsInternal( $ops ); 1277 $pathsUsed = []; 1278 foreach ( $fileOps as $fileOp ) { 1279 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() ); 1280 } 1281 1282 // Acquire any locks as needed for the scope of this function 1283 if ( empty( $opts['nonLocking'] ) ) { 1284 $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps ); 1285 /** @noinspection PhpUnusedLocalVariableInspection */ 1286 $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status ); 1287 if ( !$status->isOK() ) { 1288 return $status; // abort 1289 } 1290 } 1291 1292 // Clear any file cache entries (after locks acquired) 1293 if ( empty( $opts['preserveCache'] ) ) { 1294 $this->clearCache( $pathsUsed ); 1295 } 1296 1297 // Enlarge the cache to fit the stat entries of these files 1298 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) ); 1299 1300 // Load from the persistent container caches 1301 $this->primeContainerCache( $pathsUsed ); 1302 // Get the latest stat info for all the files (having locked them) 1303 $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] ); 1304 1305 if ( $ok ) { 1306 // Actually attempt the operation batch... 1307 $opts = $this->setConcurrencyFlags( $opts ); 1308 $subStatus = FileOpBatch::attempt( $fileOps, $opts, $this->fileJournal ); 1309 } else { 1310 // If we could not even stat some files, then bail out 1311 $subStatus = $this->newStatus( 'backend-fail-internal', $this->name ); 1312 foreach ( $ops as $i => $op ) { // mark each op as failed 1313 $subStatus->success[$i] = false; 1314 ++$subStatus->failCount; 1315 } 1316 $this->logger->error( static::class . "-{$this->name} " . 1317 " stat failure; aborted operations: " . FormatJson::encode( $ops ) ); 1318 } 1319 1320 // Merge errors into StatusValue fields 1321 $status->merge( $subStatus ); 1322 $status->success = $subStatus->success; // not done in merge() 1323 1324 // Shrink the stat cache back to normal size 1325 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE ); 1326 1327 return $status; 1328 } 1329 1330 final protected function doQuickOperationsInternal( array $ops, array $opts ) { 1331 /** @noinspection PhpUnusedLocalVariableInspection */ 1332 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1333 $status = $this->newStatus(); 1334 1335 // Fix up custom header name/value pairs 1336 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops ); 1337 // Build up a list of FileOps and involved paths 1338 $fileOps = $this->getOperationsInternal( $ops ); 1339 $pathsUsed = []; 1340 foreach ( $fileOps as $fileOp ) { 1341 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() ); 1342 } 1343 1344 // Clear any file cache entries for involved paths 1345 $this->clearCache( $pathsUsed ); 1346 1347 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen()) 1348 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); 1349 $maxConcurrency = $this->concurrency; // throttle 1350 /** @var StatusValue[] $statuses */ 1351 $statuses = []; // array of (index => StatusValue) 1352 $fileOpHandles = []; // list of (index => handle) arrays 1353 $curFileOpHandles = []; // current handle batch 1354 // Perform the sync-only ops and build up op handles for the async ops... 1355 foreach ( $fileOps as $index => $fileOp ) { 1356 $subStatus = $async 1357 ? $fileOp->attemptAsyncQuick() 1358 : $fileOp->attemptQuick(); 1359 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async 1360 if ( count( $curFileOpHandles ) >= $maxConcurrency ) { 1361 $fileOpHandles[] = $curFileOpHandles; // push this batch 1362 $curFileOpHandles = []; 1363 } 1364 $curFileOpHandles[$index] = $subStatus->value; // keep index 1365 } else { // error or completed 1366 $statuses[$index] = $subStatus; // keep index 1367 } 1368 } 1369 if ( count( $curFileOpHandles ) ) { 1370 $fileOpHandles[] = $curFileOpHandles; // last batch 1371 } 1372 // Do all the async ops that can be done concurrently... 1373 foreach ( $fileOpHandles as $fileHandleBatch ) { 1374 $statuses += $this->executeOpHandlesInternal( $fileHandleBatch ); 1375 } 1376 // Marshall and merge all the responses... 1377 foreach ( $statuses as $index => $subStatus ) { 1378 $status->merge( $subStatus ); 1379 if ( $subStatus->isOK() ) { 1380 $status->success[$index] = true; 1381 ++$status->successCount; 1382 } else { 1383 $status->success[$index] = false; 1384 ++$status->failCount; 1385 } 1386 } 1387 1388 $this->clearCache( $pathsUsed ); 1389 1390 return $status; 1391 } 1392 1393 /** 1394 * Execute a list of FileBackendStoreOpHandle handles in parallel. 1395 * The resulting StatusValue object fields will correspond 1396 * to the order in which the handles where given. 1397 * 1398 * @param FileBackendStoreOpHandle[] $fileOpHandles 1399 * @return StatusValue[] Map of StatusValue objects 1400 * @throws FileBackendError 1401 */ 1402 final public function executeOpHandlesInternal( array $fileOpHandles ) { 1403 /** @noinspection PhpUnusedLocalVariableInspection */ 1404 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1405 1406 foreach ( $fileOpHandles as $fileOpHandle ) { 1407 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { 1408 throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." ); 1409 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { 1410 throw new InvalidArgumentException( "Expected handle for this file backend." ); 1411 } 1412 } 1413 1414 $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles ); 1415 foreach ( $fileOpHandles as $fileOpHandle ) { 1416 $fileOpHandle->closeResources(); 1417 } 1418 1419 return $statuses; 1420 } 1421 1422 /** 1423 * @see FileBackendStore::executeOpHandlesInternal() 1424 * @stable to override 1425 * 1426 * @param FileBackendStoreOpHandle[] $fileOpHandles 1427 * 1428 * @throws FileBackendError 1429 * @return StatusValue[] List of corresponding StatusValue objects 1430 */ 1431 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { 1432 if ( count( $fileOpHandles ) ) { 1433 throw new FileBackendError( "Backend does not support asynchronous operations." ); 1434 } 1435 1436 return []; 1437 } 1438 1439 /** 1440 * Normalize and filter HTTP headers from a file operation 1441 * 1442 * This normalizes and strips long HTTP headers from a file operation. 1443 * Most headers are just numbers, but some are allowed to be long. 1444 * This function is useful for cleaning up headers and avoiding backend 1445 * specific errors, especially in the middle of batch file operations. 1446 * 1447 * @param array $op Same format as doOperation() 1448 * @return array 1449 */ 1450 protected function sanitizeOpHeaders( array $op ) { 1451 static $longs = [ 'content-disposition' ]; 1452 1453 if ( isset( $op['headers'] ) ) { // op sets HTTP headers 1454 $newHeaders = []; 1455 foreach ( $op['headers'] as $name => $value ) { 1456 $name = strtolower( $name ); 1457 $maxHVLen = in_array( $name, $longs ) ? INF : 255; 1458 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { 1459 $this->logger->error( "Header '{header}' is too long.", [ 1460 'filebackend' => $this->name, 1461 'header' => "$name: $value", 1462 ] ); 1463 } else { 1464 $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => "" 1465 } 1466 } 1467 $op['headers'] = $newHeaders; 1468 } 1469 1470 return $op; 1471 } 1472 1473 final public function preloadCache( array $paths ) { 1474 $fullConts = []; // full container names 1475 foreach ( $paths as $path ) { 1476 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 1477 $fullConts[] = $fullCont; 1478 } 1479 // Load from the persistent file and container caches 1480 $this->primeContainerCache( $fullConts ); 1481 $this->primeFileCache( $paths ); 1482 } 1483 1484 final public function clearCache( array $paths = null ) { 1485 if ( is_array( $paths ) ) { 1486 $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths ); 1487 $paths = array_filter( $paths, 'strlen' ); // remove nulls 1488 } 1489 if ( $paths === null ) { 1490 $this->cheapCache->clear(); 1491 $this->expensiveCache->clear(); 1492 } else { 1493 foreach ( $paths as $path ) { 1494 $this->cheapCache->clear( $path ); 1495 $this->expensiveCache->clear( $path ); 1496 } 1497 } 1498 $this->doClearCache( $paths ); 1499 } 1500 1501 /** 1502 * Clears any additional stat caches for storage paths 1503 * @stable to override 1504 * 1505 * @see FileBackend::clearCache() 1506 * 1507 * @param array|null $paths Storage paths (optional) 1508 */ 1509 protected function doClearCache( array $paths = null ) { 1510 } 1511 1512 final public function preloadFileStat( array $params ) { 1513 /** @noinspection PhpUnusedLocalVariableInspection */ 1514 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1515 1516 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1; 1517 $stats = $this->doGetFileStatMulti( $params ); 1518 if ( $stats === null ) { 1519 return true; // not supported 1520 } 1521 1522 // Whether this queried the backend in high consistency mode 1523 $latest = !empty( $params['latest'] ); 1524 1525 return $this->ingestFreshFileStats( $stats, $latest ); 1526 } 1527 1528 /** 1529 * Get file stat information (concurrently if possible) for several files 1530 * @stable to override 1531 * 1532 * @see FileBackend::getFileStat() 1533 * 1534 * @param array $params Parameters include: 1535 * - srcs : list of source storage paths 1536 * - latest : use the latest available data 1537 * @return array|null Map of storage paths to array|bool|null (returns null if not supported) 1538 * @since 1.23 1539 */ 1540 protected function doGetFileStatMulti( array $params ) { 1541 return null; // not supported 1542 } 1543 1544 /** 1545 * Is this a key/value store where directories are just virtual? 1546 * Virtual directories exists in so much as files exists that are 1547 * prefixed with the directory path followed by a forward slash. 1548 * 1549 * @return bool 1550 */ 1551 abstract protected function directoriesAreVirtual(); 1552 1553 /** 1554 * Check if a short container name is valid 1555 * 1556 * This checks for length and illegal characters. 1557 * This may disallow certain characters that can appear 1558 * in the prefix used to make the full container name. 1559 * 1560 * @param string $container 1561 * @return bool 1562 */ 1563 final protected static function isValidShortContainerName( $container ) { 1564 // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments) 1565 // might be used by subclasses. Reserve the dot character for sanity. 1566 // The only way dots end up in containers (e.g. resolveStoragePath) 1567 // is due to the wikiId container prefix or the above suffixes. 1568 return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container ); 1569 } 1570 1571 /** 1572 * Check if a full container name is valid 1573 * 1574 * This checks for length and illegal characters. 1575 * Limiting the characters makes migrations to other stores easier. 1576 * 1577 * @param string $container 1578 * @return bool 1579 */ 1580 final protected static function isValidContainerName( $container ) { 1581 // This accounts for NTFS, Swift, and Ceph restrictions 1582 // and disallows directory separators or traversal characters. 1583 // Note that matching strings URL encode to the same string; 1584 // in Swift/Ceph, the length restriction is *after* URL encoding. 1585 return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container ); 1586 } 1587 1588 /** 1589 * Splits a storage path into an internal container name, 1590 * an internal relative file name, and a container shard suffix. 1591 * Any shard suffix is already appended to the internal container name. 1592 * This also checks that the storage path is valid and within this backend. 1593 * 1594 * If the container is sharded but a suffix could not be determined, 1595 * this means that the path can only refer to a directory and can only 1596 * be scanned by looking in all the container shards. 1597 * 1598 * @param string $storagePath 1599 * @return array (container, path, container suffix) or (null, null, null) if invalid 1600 */ 1601 final protected function resolveStoragePath( $storagePath ) { 1602 list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath ); 1603 if ( $backend === $this->name ) { // must be for this backend 1604 $relPath = self::normalizeContainerPath( $relPath ); 1605 if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) { 1606 // Get shard for the normalized path if this container is sharded 1607 $cShard = $this->getContainerShard( $shortCont, $relPath ); 1608 // Validate and sanitize the relative path (backend-specific) 1609 $relPath = $this->resolveContainerPath( $shortCont, $relPath ); 1610 if ( $relPath !== null ) { 1611 // Prepend any domain ID prefix to the container name 1612 $container = $this->fullContainerName( $shortCont ); 1613 if ( self::isValidContainerName( $container ) ) { 1614 // Validate and sanitize the container name (backend-specific) 1615 $container = $this->resolveContainerName( "{$container}{$cShard}" ); 1616 if ( $container !== null ) { 1617 return [ $container, $relPath, $cShard ]; 1618 } 1619 } 1620 } 1621 } 1622 } 1623 1624 return [ null, null, null ]; 1625 } 1626 1627 /** 1628 * Like resolveStoragePath() except null values are returned if 1629 * the container is sharded and the shard could not be determined 1630 * or if the path ends with '/'. The latter case is illegal for FS 1631 * backends and can confuse listings for object store backends. 1632 * 1633 * This function is used when resolving paths that must be valid 1634 * locations for files. Directory and listing functions should 1635 * generally just use resolveStoragePath() instead. 1636 * 1637 * @see FileBackendStore::resolveStoragePath() 1638 * 1639 * @param string $storagePath 1640 * @return array (container, path) or (null, null) if invalid 1641 */ 1642 final protected function resolveStoragePathReal( $storagePath ) { 1643 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); 1644 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { 1645 return [ $container, $relPath ]; 1646 } 1647 1648 return [ null, null ]; 1649 } 1650 1651 /** 1652 * Get the container name shard suffix for a given path. 1653 * Any empty suffix means the container is not sharded. 1654 * 1655 * @param string $container Container name 1656 * @param string $relPath Storage path relative to the container 1657 * @return string|null Returns null if shard could not be determined 1658 */ 1659 final protected function getContainerShard( $container, $relPath ) { 1660 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); 1661 if ( $levels == 1 || $levels == 2 ) { 1662 // Hash characters are either base 16 or 36 1663 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; 1664 // Get a regex that represents the shard portion of paths. 1665 // The concatenation of the captures gives us the shard. 1666 if ( $levels === 1 ) { // 16 or 36 shards per container 1667 $hashDirRegex = '(' . $char . ')'; 1668 } else { // 256 or 1296 shards per container 1669 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") 1670 $hashDirRegex = $char . '/(' . $char . '{2})'; 1671 } else { // short hash dir format (e.g. "a/b/c") 1672 $hashDirRegex = '(' . $char . ')/(' . $char . ')'; 1673 } 1674 } 1675 // Allow certain directories to be above the hash dirs so as 1676 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). 1677 // They must be 2+ chars to avoid any hash directory ambiguity. 1678 $m = []; 1679 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { 1680 return '.' . implode( '', array_slice( $m, 1 ) ); 1681 } 1682 1683 return null; // failed to match 1684 } 1685 1686 return ''; // no sharding 1687 } 1688 1689 /** 1690 * Check if a storage path maps to a single shard. 1691 * Container dirs like "a", where the container shards on "x/xy", 1692 * can reside on several shards. Such paths are tricky to handle. 1693 * 1694 * @param string $storagePath Storage path 1695 * @return bool 1696 */ 1697 final public function isSingleShardPathInternal( $storagePath ) { 1698 list( , , $shard ) = $this->resolveStoragePath( $storagePath ); 1699 1700 return ( $shard !== null ); 1701 } 1702 1703 /** 1704 * Get the sharding config for a container. 1705 * If greater than 0, then all file storage paths within 1706 * the container are required to be hashed accordingly. 1707 * 1708 * @param string $container 1709 * @return array (integer levels, integer base, repeat flag) or (0, 0, false) 1710 */ 1711 final protected function getContainerHashLevels( $container ) { 1712 if ( isset( $this->shardViaHashLevels[$container] ) ) { 1713 $config = $this->shardViaHashLevels[$container]; 1714 $hashLevels = (int)$config['levels']; 1715 if ( $hashLevels == 1 || $hashLevels == 2 ) { 1716 $hashBase = (int)$config['base']; 1717 if ( $hashBase == 16 || $hashBase == 36 ) { 1718 return [ $hashLevels, $hashBase, $config['repeat'] ]; 1719 } 1720 } 1721 } 1722 1723 return [ 0, 0, false ]; // no sharding 1724 } 1725 1726 /** 1727 * Get a list of full container shard suffixes for a container 1728 * 1729 * @param string $container 1730 * @return array 1731 */ 1732 final protected function getContainerSuffixes( $container ) { 1733 $shards = []; 1734 list( $digits, $base ) = $this->getContainerHashLevels( $container ); 1735 if ( $digits > 0 ) { 1736 $numShards = $base ** $digits; 1737 for ( $index = 0; $index < $numShards; $index++ ) { 1738 $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits ); 1739 } 1740 } 1741 1742 return $shards; 1743 } 1744 1745 /** 1746 * Get the full container name, including the domain ID prefix 1747 * 1748 * @param string $container 1749 * @return string 1750 */ 1751 final protected function fullContainerName( $container ) { 1752 if ( $this->domainId != '' ) { 1753 return "{$this->domainId}-$container"; 1754 } else { 1755 return $container; 1756 } 1757 } 1758 1759 /** 1760 * Resolve a container name, checking if it's allowed by the backend. 1761 * This is intended for internal use, such as encoding illegal chars. 1762 * Subclasses can override this to be more restrictive. 1763 * @stable to override 1764 * 1765 * @param string $container 1766 * @return string|null 1767 */ 1768 protected function resolveContainerName( $container ) { 1769 return $container; 1770 } 1771 1772 /** 1773 * Resolve a relative storage path, checking if it's allowed by the backend. 1774 * This is intended for internal use, such as encoding illegal chars or perhaps 1775 * getting absolute paths (e.g. FS based backends). Note that the relative path 1776 * may be the empty string (e.g. the path is simply to the container). 1777 * @stable to override 1778 * 1779 * @param string $container Container name 1780 * @param string $relStoragePath Storage path relative to the container 1781 * @return string|null Path or null if not valid 1782 */ 1783 protected function resolveContainerPath( $container, $relStoragePath ) { 1784 return $relStoragePath; 1785 } 1786 1787 /** 1788 * Get the cache key for a container 1789 * 1790 * @param string $container Resolved container name 1791 * @return string 1792 */ 1793 private function containerCacheKey( $container ) { 1794 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}"; 1795 } 1796 1797 /** 1798 * Set the cached info for a container 1799 * 1800 * @param string $container Resolved container name 1801 * @param array $val Information to cache 1802 */ 1803 final protected function setContainerCache( $container, array $val ) { 1804 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ); 1805 } 1806 1807 /** 1808 * Delete the cached info for a container. 1809 * The cache key is salted for a while to prevent race conditions. 1810 * 1811 * @param string $container Resolved container name 1812 */ 1813 final protected function deleteContainerCache( $container ) { 1814 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) { 1815 $this->logger->warning( "Unable to delete stat cache for container {container}.", 1816 [ 'filebackend' => $this->name, 'container' => $container ] 1817 ); 1818 } 1819 } 1820 1821 /** 1822 * Do a batch lookup from cache for container stats for all containers 1823 * used in a list of container names or storage paths objects. 1824 * This loads the persistent cache values into the process cache. 1825 * 1826 * @param array $items 1827 */ 1828 final protected function primeContainerCache( array $items ) { 1829 /** @noinspection PhpUnusedLocalVariableInspection */ 1830 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1831 1832 $paths = []; // list of storage paths 1833 $contNames = []; // (cache key => resolved container name) 1834 // Get all the paths/containers from the items... 1835 foreach ( $items as $item ) { 1836 if ( self::isStoragePath( $item ) ) { 1837 $paths[] = $item; 1838 } elseif ( is_string( $item ) ) { // full container name 1839 $contNames[$this->containerCacheKey( $item )] = $item; 1840 } 1841 } 1842 // Get all the corresponding cache keys for paths... 1843 foreach ( $paths as $path ) { 1844 list( $fullCont, , ) = $this->resolveStoragePath( $path ); 1845 if ( $fullCont !== null ) { // valid path for this backend 1846 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; 1847 } 1848 } 1849 1850 $contInfo = []; // (resolved container name => cache value) 1851 // Get all cache entries for these container cache keys... 1852 $values = $this->memCache->getMulti( array_keys( $contNames ) ); 1853 foreach ( $values as $cacheKey => $val ) { 1854 $contInfo[$contNames[$cacheKey]] = $val; 1855 } 1856 1857 // Populate the container process cache for the backend... 1858 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); 1859 } 1860 1861 /** 1862 * Fill the backend-specific process cache given an array of 1863 * resolved container names and their corresponding cached info. 1864 * Only containers that actually exist should appear in the map. 1865 * @stable to override 1866 * 1867 * @param array $containerInfo Map of resolved container names to cached info 1868 */ 1869 protected function doPrimeContainerCache( array $containerInfo ) { 1870 } 1871 1872 /** 1873 * Get the cache key for a file path 1874 * 1875 * @param string $path Normalized storage path 1876 * @return string 1877 */ 1878 private function fileCacheKey( $path ) { 1879 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path ); 1880 } 1881 1882 /** 1883 * Set the cached stat info for a file path. 1884 * Negatives (404s) are not cached. By not caching negatives, we can skip cache 1885 * salting for the case when a file is created at a path were there was none before. 1886 * 1887 * @param string $path Storage path 1888 * @param array $val Stat information to cache 1889 */ 1890 final protected function setFileCache( $path, array $val ) { 1891 $path = FileBackend::normalizeStoragePath( $path ); 1892 if ( $path === null ) { 1893 return; // invalid storage path 1894 } 1895 $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] ); 1896 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 ); 1897 $key = $this->fileCacheKey( $path ); 1898 // Set the cache unless it is currently salted. 1899 $this->memCache->set( $key, $val, $ttl ); 1900 } 1901 1902 /** 1903 * Delete the cached stat info for a file path. 1904 * The cache key is salted for a while to prevent race conditions. 1905 * Since negatives (404s) are not cached, this does not need to be called when 1906 * a file is created at a path were there was none before. 1907 * 1908 * @param string $path Storage path 1909 */ 1910 final protected function deleteFileCache( $path ) { 1911 $path = FileBackend::normalizeStoragePath( $path ); 1912 if ( $path === null ) { 1913 return; // invalid storage path 1914 } 1915 if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) { 1916 $this->logger->warning( "Unable to delete stat cache for file {path}.", 1917 [ 'filebackend' => $this->name, 'path' => $path ] 1918 ); 1919 } 1920 } 1921 1922 /** 1923 * Do a batch lookup from cache for file stats for all paths 1924 * used in a list of storage paths or FileOp objects. 1925 * This loads the persistent cache values into the process cache. 1926 * 1927 * @param array $items List of storage paths 1928 */ 1929 final protected function primeFileCache( array $items ) { 1930 /** @noinspection PhpUnusedLocalVariableInspection */ 1931 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); 1932 1933 $paths = []; // list of storage paths 1934 $pathNames = []; // (cache key => storage path) 1935 // Get all the paths/containers from the items... 1936 foreach ( $items as $item ) { 1937 if ( self::isStoragePath( $item ) ) { 1938 $paths[] = FileBackend::normalizeStoragePath( $item ); 1939 } 1940 } 1941 // Get rid of any paths that failed normalization 1942 $paths = array_filter( $paths, 'strlen' ); // remove nulls 1943 // Get all the corresponding cache keys for paths... 1944 foreach ( $paths as $path ) { 1945 list( , $rel, ) = $this->resolveStoragePath( $path ); 1946 if ( $rel !== null ) { // valid path for this backend 1947 $pathNames[$this->fileCacheKey( $path )] = $path; 1948 } 1949 } 1950 // Get all cache entries for these file cache keys. 1951 // Note that negatives are not cached by getFileStat()/preloadFileStat(). 1952 $values = $this->memCache->getMulti( array_keys( $pathNames ) ); 1953 // Load all of the results into process cache... 1954 foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) { 1955 $path = $pathNames[$cacheKey]; 1956 // Sanity; this flag only applies to stat info loaded directly 1957 // from a high consistency backend query to the process cache 1958 unset( $stat['latest'] ); 1959 1960 $this->cheapCache->setField( $path, 'stat', $stat ); 1961 if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) { 1962 // Some backends store SHA-1 as metadata 1963 $this->cheapCache->setField( 1964 $path, 1965 'sha1', 1966 [ 'hash' => $stat['sha1'], 'latest' => false ] 1967 ); 1968 } 1969 if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) { 1970 // Some backends store custom headers/metadata 1971 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); 1972 $this->cheapCache->setField( 1973 $path, 1974 'xattr', 1975 [ 'map' => $stat['xattr'], 'latest' => false ] 1976 ); 1977 } 1978 } 1979 } 1980 1981 /** 1982 * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format 1983 * 1984 * @param array $xattr 1985 * @return array 1986 * @since 1.22 1987 */ 1988 final protected static function normalizeXAttributes( array $xattr ) { 1989 $newXAttr = [ 'headers' => [], 'metadata' => [] ]; 1990 1991 foreach ( $xattr['headers'] as $name => $value ) { 1992 $newXAttr['headers'][strtolower( $name )] = $value; 1993 } 1994 1995 foreach ( $xattr['metadata'] as $name => $value ) { 1996 $newXAttr['metadata'][strtolower( $name )] = $value; 1997 } 1998 1999 return $newXAttr; 2000 } 2001 2002 /** 2003 * Set the 'concurrency' option from a list of operation options 2004 * 2005 * @param array $opts Map of operation options 2006 * @return array 2007 */ 2008 final protected function setConcurrencyFlags( array $opts ) { 2009 $opts['concurrency'] = 1; // off 2010 if ( $this->parallelize === 'implicit' ) { 2011 if ( $opts['parallelize'] ?? true ) { 2012 $opts['concurrency'] = $this->concurrency; 2013 } 2014 } elseif ( $this->parallelize === 'explicit' ) { 2015 if ( !empty( $opts['parallelize'] ) ) { 2016 $opts['concurrency'] = $this->concurrency; 2017 } 2018 } 2019 2020 return $opts; 2021 } 2022 2023 /** 2024 * Get the content type to use in HEAD/GET requests for a file 2025 * @stable to override 2026 * 2027 * @param string $storagePath 2028 * @param string|null $content File data 2029 * @param string|null $fsPath File system path 2030 * @return string MIME type 2031 */ 2032 protected function getContentType( $storagePath, $content, $fsPath ) { 2033 if ( $this->mimeCallback ) { 2034 return call_user_func_array( $this->mimeCallback, func_get_args() ); 2035 } 2036 2037 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false; 2038 return $mime ?: 'unknown/unknown'; 2039 } 2040} 2041