1<?php 2 3use MediaWiki\MediaWikiServices; 4 5/** 6 * Backend for uploading files from chunks. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, write to the Free Software Foundation, Inc., 20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 * http://www.gnu.org/copyleft/gpl.html 22 * 23 * @file 24 * @ingroup Upload 25 */ 26 27/** 28 * Implements uploading from chunks 29 * 30 * @ingroup Upload 31 * @author Michael Dale 32 */ 33class UploadFromChunks extends UploadFromFile { 34 /** @var LocalRepo */ 35 private $repo; 36 /** @var UploadStash */ 37 public $stash; 38 /** @var User */ 39 public $user; 40 41 protected $mOffset; 42 protected $mChunkIndex; 43 protected $mFileKey; 44 protected $mVirtualTempPath; 45 46 /** @noinspection PhpMissingParentConstructorInspection */ 47 48 /** 49 * Setup local pointers to stash, repo and user (similar to UploadFromStash) 50 * 51 * @param User $user 52 * @param UploadStash|bool $stash Default: false 53 * @param FileRepo|bool $repo Default: false 54 */ 55 public function __construct( User $user, $stash = false, $repo = false ) { 56 $this->user = $user; 57 58 if ( $repo ) { 59 $this->repo = $repo; 60 } else { 61 $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo(); 62 } 63 64 if ( $stash ) { 65 $this->stash = $stash; 66 } else { 67 wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() ); 68 $this->stash = new UploadStash( $this->repo, $this->user ); 69 } 70 } 71 72 /** 73 * @inheritDoc 74 */ 75 public function tryStashFile( User $user, $isPartial = false ) { 76 try { 77 $this->verifyChunk(); 78 } catch ( UploadChunkVerificationException $e ) { 79 return Status::newFatal( $e->msg ); 80 } 81 82 return parent::tryStashFile( $user, $isPartial ); 83 } 84 85 /** 86 * @inheritDoc 87 * @throws UploadChunkVerificationException 88 * @deprecated since 1.28 Use tryStashFile() instead 89 */ 90 public function stashFile( User $user = null ) { 91 wfDeprecated( __METHOD__, '1.28' ); 92 93 $this->verifyChunk(); 94 return parent::stashFile( $user ); 95 } 96 97 /** 98 * Calls the parent doStashFile and updates the uploadsession table to handle "chunks" 99 * 100 * @param User|null $user 101 * @return UploadStashFile Stashed file 102 */ 103 protected function doStashFile( User $user = null ) { 104 // Stash file is the called on creating a new chunk session: 105 $this->mChunkIndex = 0; 106 $this->mOffset = 0; 107 108 // Create a local stash target 109 $this->mStashFile = parent::doStashFile( $user ); 110 // Update the initial file offset (based on file size) 111 $this->mOffset = $this->mStashFile->getSize(); 112 $this->mFileKey = $this->mStashFile->getFileKey(); 113 114 // Output a copy of this first to chunk 0 location: 115 $this->outputChunk( $this->mStashFile->getPath() ); 116 117 // Update db table to reflect initial "chunk" state 118 $this->updateChunkStatus(); 119 120 return $this->mStashFile; 121 } 122 123 /** 124 * Continue chunk uploading 125 * 126 * @param string $name 127 * @param string $key 128 * @param WebRequestUpload $webRequestUpload 129 */ 130 public function continueChunks( $name, $key, $webRequestUpload ) { 131 $this->mFileKey = $key; 132 $this->mUpload = $webRequestUpload; 133 // Get the chunk status form the db: 134 $this->getChunkStatus(); 135 136 $metadata = $this->stash->getMetadata( $key ); 137 $this->initializePathInfo( $name, 138 $this->getRealPath( $metadata['us_path'] ), 139 $metadata['us_size'], 140 false 141 ); 142 } 143 144 /** 145 * Append the final chunk and ready file for parent::performUpload() 146 * @return Status 147 */ 148 public function concatenateChunks() { 149 $chunkIndex = $this->getChunkIndex(); 150 wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" . 151 $this->getOffset() . ' inx:' . $chunkIndex ); 152 153 // Concatenate all the chunks to mVirtualTempPath 154 $fileList = []; 155 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1" 156 for ( $i = 0; $i <= $chunkIndex; $i++ ) { 157 $fileList[] = $this->getVirtualChunkLocation( $i ); 158 } 159 160 // Get the file extension from the last chunk 161 $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath ); 162 // Get a 0-byte temp file to perform the concatenation at 163 $tmpFile = MediaWikiServices::getInstance()->getTempFSFileFactory() 164 ->newTempFSFile( 'chunkedupload_', $ext ); 165 $tmpPath = false; // fail in concatenate() 166 if ( $tmpFile ) { 167 // keep alive with $this 168 $tmpPath = $tmpFile->bind( $this )->getPath(); 169 } 170 171 // Concatenate the chunks at the temp file 172 $tStart = microtime( true ); 173 $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE ); 174 $tAmount = microtime( true ) - $tStart; 175 if ( !$status->isOK() ) { 176 return $status; 177 } 178 179 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." ); 180 181 // File system path of the actual full temp file 182 $this->setTempFile( $tmpPath ); 183 184 $ret = $this->verifyUpload(); 185 if ( $ret['status'] !== UploadBase::OK ) { 186 wfDebugLog( 'fileconcatenate', "Verification failed for chunked upload" ); 187 $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) ); 188 189 return $status; 190 } 191 192 // Update the mTempPath and mStashFile 193 // (for FileUpload or normal Stash to take over) 194 $tStart = microtime( true ); 195 // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we 196 // override doStashFile() with completely different functionality in this class... 197 $error = $this->runUploadStashFileHook( $this->user ); 198 if ( $error ) { 199 $status->fatal( ...$error ); 200 return $status; 201 } 202 try { 203 $this->mStashFile = parent::doStashFile( $this->user ); 204 } catch ( UploadStashException $e ) { 205 $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() ); 206 return $status; 207 } 208 209 $tAmount = microtime( true ) - $tStart; 210 $this->mStashFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo()) 211 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." ); 212 213 return $status; 214 } 215 216 /** 217 * Returns the virtual chunk location: 218 * @param int $index 219 * @return string 220 */ 221 private function getVirtualChunkLocation( $index ) { 222 return $this->repo->getVirtualUrl( 'temp' ) . 223 '/' . 224 $this->repo->getHashPath( 225 $this->getChunkFileKey( $index ) 226 ) . 227 $this->getChunkFileKey( $index ); 228 } 229 230 /** 231 * Add a chunk to the temporary directory 232 * 233 * @param string $chunkPath Path to temporary chunk file 234 * @param int $chunkSize Size of the current chunk 235 * @param int $offset Offset of current chunk ( mutch match database chunk offset ) 236 * @return Status 237 */ 238 public function addChunk( $chunkPath, $chunkSize, $offset ) { 239 // Get the offset before we add the chunk to the file system 240 $preAppendOffset = $this->getOffset(); 241 242 if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize() ) { 243 $status = Status::newFatal( 'file-too-large' ); 244 } else { 245 // Make sure the client is uploading the correct chunk with a matching offset. 246 if ( $preAppendOffset == $offset ) { 247 // Update local chunk index for the current chunk 248 $this->mChunkIndex++; 249 try { 250 # For some reason mTempPath is set to first part 251 $oldTemp = $this->mTempPath; 252 $this->mTempPath = $chunkPath; 253 $this->verifyChunk(); 254 $this->mTempPath = $oldTemp; 255 } catch ( UploadChunkVerificationException $e ) { 256 return Status::newFatal( $e->msg ); 257 } 258 $status = $this->outputChunk( $chunkPath ); 259 if ( $status->isGood() ) { 260 // Update local offset: 261 $this->mOffset = $preAppendOffset + $chunkSize; 262 // Update chunk table status db 263 $this->updateChunkStatus(); 264 } 265 } else { 266 $status = Status::newFatal( 'invalid-chunk-offset' ); 267 } 268 } 269 270 return $status; 271 } 272 273 /** 274 * Update the chunk db table with the current status: 275 */ 276 private function updateChunkStatus() { 277 wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" . 278 $this->getOffset() . ' inx:' . $this->getChunkIndex() ); 279 280 $dbw = $this->repo->getMasterDB(); 281 $dbw->update( 282 'uploadstash', 283 [ 284 'us_status' => 'chunks', 285 'us_chunk_inx' => $this->getChunkIndex(), 286 'us_size' => $this->getOffset() 287 ], 288 [ 'us_key' => $this->mFileKey ], 289 __METHOD__ 290 ); 291 } 292 293 /** 294 * Get the chunk db state and populate update relevant local values 295 */ 296 private function getChunkStatus() { 297 // get Master db to avoid race conditions. 298 // Otherwise, if chunk upload time < replag there will be spurious errors 299 $dbw = $this->repo->getMasterDB(); 300 $row = $dbw->selectRow( 301 'uploadstash', 302 [ 303 'us_chunk_inx', 304 'us_size', 305 'us_path', 306 ], 307 [ 'us_key' => $this->mFileKey ], 308 __METHOD__ 309 ); 310 // Handle result: 311 if ( $row ) { 312 $this->mChunkIndex = $row->us_chunk_inx; 313 $this->mOffset = $row->us_size; 314 $this->mVirtualTempPath = $row->us_path; 315 } 316 } 317 318 /** 319 * Get the current Chunk index 320 * @return int Index of the current chunk 321 */ 322 private function getChunkIndex() { 323 if ( $this->mChunkIndex !== null ) { 324 return $this->mChunkIndex; 325 } 326 327 return 0; 328 } 329 330 /** 331 * Get the offset at which the next uploaded chunk will be appended to 332 * @return int Current byte offset of the chunk file set 333 */ 334 public function getOffset() { 335 if ( $this->mOffset !== null ) { 336 return $this->mOffset; 337 } 338 339 return 0; 340 } 341 342 /** 343 * Output the chunk to disk 344 * 345 * @param string $chunkPath 346 * @throws UploadChunkFileException 347 * @return Status 348 */ 349 private function outputChunk( $chunkPath ) { 350 // Key is fileKey + chunk index 351 $fileKey = $this->getChunkFileKey(); 352 353 // Store the chunk per its indexed fileKey: 354 $hashPath = $this->repo->getHashPath( $fileKey ); 355 $storeStatus = $this->repo->quickImport( $chunkPath, 356 $this->repo->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" ); 357 358 // Check for error in stashing the chunk: 359 if ( !$storeStatus->isOK() ) { 360 $error = $storeStatus->getErrorsArray(); 361 $error = reset( $error ); 362 if ( !count( $error ) ) { 363 $error = $storeStatus->getWarningsArray(); 364 $error = reset( $error ); 365 if ( !count( $error ) ) { 366 $error = [ 'unknown', 'no error recorded' ]; 367 } 368 } 369 throw new UploadChunkFileException( "Error storing file in '$chunkPath': " . 370 implode( '; ', $error ) ); 371 } 372 373 return $storeStatus; 374 } 375 376 private function getChunkFileKey( $index = null ) { 377 if ( $index === null ) { 378 $index = $this->getChunkIndex(); 379 } 380 381 return $this->mFileKey . '.' . $index; 382 } 383 384 /** 385 * Verify that the chunk isn't really an evil html file 386 * 387 * @throws UploadChunkVerificationException 388 */ 389 private function verifyChunk() { 390 // Rest mDesiredDestName here so we verify the name as if it were mFileKey 391 $oldDesiredDestName = $this->mDesiredDestName; 392 $this->mDesiredDestName = $this->mFileKey; 393 $this->mTitle = false; 394 $res = $this->verifyPartialFile(); 395 $this->mDesiredDestName = $oldDesiredDestName; 396 $this->mTitle = false; 397 if ( is_array( $res ) ) { 398 throw new UploadChunkVerificationException( $res ); 399 } 400 } 401} 402