1<?php 2/** 3 * This program is free software; you can redistribute it and/or modify 4 * it under the terms of the GNU General Public License as published by 5 * the Free Software Foundation; either version 2 of the License, or 6 * (at your option) any later version. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License along 14 * with this program; if not, write to the Free Software Foundation, Inc., 15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 16 * http://www.gnu.org/copyleft/gpl.html 17 * 18 * @file 19 * @ingroup Maintenance 20 */ 21 22use MediaWiki\MediaWikiServices; 23use Wikimedia\AtEase\AtEase; 24 25/** 26 * Manage foreign resources registered with ResourceLoader. 27 * 28 * @since 1.32 29 */ 30class ForeignResourceManager { 31 private $defaultAlgo = 'sha384'; 32 private $hasErrors = false; 33 private $registryFile; 34 private $libDir; 35 private $tmpParentDir; 36 private $cacheDir; 37 /** 38 * @var callable|Closure 39 * @phan-var callable(string):void 40 */ 41 private $infoPrinter; 42 /** 43 * @var callable|Closure 44 * @phan-var callable(string):void 45 */ 46 private $errorPrinter; 47 /** 48 * @var callable|Closure 49 * @phan-var callable(string):void 50 */ 51 private $verbosePrinter; 52 private $action; 53 /** @var array[] */ 54 private $registry; 55 56 /** 57 * @param string $registryFile Path to YAML file 58 * @param string $libDir Path to a modules directory 59 * @param callable|null $infoPrinter Callback for printing info about the run. 60 * @param callable|null $errorPrinter Callback for printing errors from the run. 61 * @param callable|null $verbosePrinter Callback for printing extra verbose 62 * progress information from the run. 63 */ 64 public function __construct( 65 $registryFile, 66 $libDir, 67 callable $infoPrinter = null, 68 callable $errorPrinter = null, 69 callable $verbosePrinter = null 70 ) { 71 $this->registryFile = $registryFile; 72 $this->libDir = $libDir; 73 $this->infoPrinter = $infoPrinter ?? function ( $_ ) { 74 }; 75 $this->errorPrinter = $errorPrinter ?? $this->infoPrinter; 76 $this->verbosePrinter = $verbosePrinter ?? function ( $_ ) { 77 }; 78 79 // Use a temporary directory under the destination directory instead 80 // of wfTempDir() because PHP's rename() does not work across file 81 // systems, and the user's /tmp and $IP may be on different filesystems. 82 $this->tmpParentDir = "{$this->libDir}/.foreign/tmp"; 83 84 $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false; 85 $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache"; 86 } 87 88 /** 89 * @param string $action 90 * @param string $module 91 * @return bool 92 * @throws Exception 93 */ 94 public function run( $action, $module ) { 95 $actions = [ 'update', 'verify', 'make-sri' ]; 96 if ( !in_array( $action, $actions ) ) { 97 $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' ); 98 return false; 99 } 100 $this->action = $action; 101 102 $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) ); 103 if ( $module === 'all' ) { 104 $modules = $this->registry; 105 } elseif ( isset( $this->registry[ $module ] ) ) { 106 $modules = [ $module => $this->registry[ $module ] ]; 107 } else { 108 $this->error( "Unknown module name.\n\nMust be one of:\n" . 109 wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) . 110 '.' 111 ); 112 return false; 113 } 114 115 foreach ( $modules as $moduleName => $info ) { 116 $this->verbose( "\n### {$moduleName}\n\n" ); 117 $destDir = "{$this->libDir}/$moduleName"; 118 119 if ( $this->action === 'update' ) { 120 $this->output( "... updating '{$moduleName}'\n" ); 121 $this->verbose( "... emptying directory for $moduleName\n" ); 122 wfRecursiveRemoveDir( $destDir ); 123 } elseif ( $this->action === 'verify' ) { 124 $this->output( "... verifying '{$moduleName}'\n" ); 125 } else { 126 $this->output( "... checking '{$moduleName}'\n" ); 127 } 128 129 $this->verbose( "... preparing {$this->tmpParentDir}\n" ); 130 wfRecursiveRemoveDir( $this->tmpParentDir ); 131 if ( !wfMkdirParents( $this->tmpParentDir ) ) { 132 throw new Exception( "Unable to create {$this->tmpParentDir}" ); 133 } 134 135 if ( !isset( $info['type'] ) ) { 136 throw new Exception( "Module '$moduleName' must have a 'type' key." ); 137 } 138 switch ( $info['type'] ) { 139 case 'tar': 140 $this->handleTypeTar( $moduleName, $destDir, $info ); 141 break; 142 case 'file': 143 $this->handleTypeFile( $moduleName, $destDir, $info ); 144 break; 145 case 'multi-file': 146 $this->handleTypeMultiFile( $moduleName, $destDir, $info ); 147 break; 148 default: 149 throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" ); 150 } 151 } 152 153 $this->output( "\nDone!\n" ); 154 $this->cleanUp(); 155 if ( $this->hasErrors ) { 156 // The verify mode should check all modules/files and fail after, not during. 157 return false; 158 } 159 160 return true; 161 } 162 163 private function cacheKey( $src, $integrity ) { 164 $key = basename( $src ) . '_' . substr( $integrity, -12 ); 165 $key = preg_replace( '/[.\/+?=_-]+/', '_', $key ); 166 return rtrim( $key, '_' ); 167 } 168 169 /** 170 * @param string $key 171 * @return string|false 172 */ 173 private function cacheGet( $key ) { 174 return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" ); 175 } 176 177 private function cacheSet( $key, $data ) { 178 wfMkdirParents( $this->cacheDir ); 179 file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX ); 180 } 181 182 private function fetch( $src, $integrity ) { 183 $key = $this->cacheKey( $src, $integrity ); 184 $data = $this->cacheGet( $key ); 185 if ( $data ) { 186 return $data; 187 } 188 189 $req = MediaWikiServices::getInstance()->getHttpRequestFactory() 190 ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ ); 191 if ( !$req->execute()->isOK() ) { 192 throw new Exception( "Failed to download resource at {$src}" ); 193 } 194 if ( $req->getStatus() !== 200 ) { 195 throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" ); 196 } 197 $data = $req->getContent(); 198 $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; 199 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); 200 if ( $integrity === $actualIntegrity ) { 201 $this->verbose( "... passed integrity check for {$src}\n" ); 202 $this->cacheSet( $key, $data ); 203 } elseif ( $this->action === 'make-sri' ) { 204 $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" ); 205 } else { 206 throw new Exception( "Integrity check failed for {$src}\n" . 207 "\tExpected: {$integrity}\n" . 208 "\tActual: {$actualIntegrity}" 209 ); 210 } 211 return $data; 212 } 213 214 private function handleTypeFile( $moduleName, $destDir, array $info ) { 215 if ( !isset( $info['src'] ) ) { 216 throw new Exception( "Module '$moduleName' must have a 'src' key." ); 217 } 218 $data = $this->fetch( $info['src'], $info['integrity'] ?? null ); 219 $dest = $info['dest'] ?? basename( $info['src'] ); 220 $path = "$destDir/$dest"; 221 if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { 222 throw new Exception( "File for '$moduleName' is different." ); 223 } 224 if ( $this->action === 'update' ) { 225 wfMkdirParents( $destDir ); 226 file_put_contents( "$destDir/$dest", $data ); 227 } 228 } 229 230 private function handleTypeMultiFile( $moduleName, $destDir, array $info ) { 231 if ( !isset( $info['files'] ) ) { 232 throw new Exception( "Module '$moduleName' must have a 'files' key." ); 233 } 234 foreach ( $info['files'] as $dest => $file ) { 235 if ( !isset( $file['src'] ) ) { 236 throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." ); 237 } 238 $data = $this->fetch( $file['src'], $file['integrity'] ?? null ); 239 $path = "$destDir/$dest"; 240 if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { 241 throw new Exception( "File '$dest' for '$moduleName' is different." ); 242 } elseif ( $this->action === 'update' ) { 243 wfMkdirParents( $destDir ); 244 file_put_contents( "$destDir/$dest", $data ); 245 } 246 } 247 } 248 249 private function handleTypeTar( $moduleName, $destDir, array $info ) { 250 $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; 251 if ( $info['src'] === null ) { 252 throw new Exception( "Module '$moduleName' must have a 'src' key." ); 253 } 254 // Download the resource to a temporary file and open it 255 $data = $this->fetch( $info['src'], $info['integrity' ] ); 256 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; 257 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); 258 file_put_contents( $tmpFile, $data ); 259 $p = new PharData( $tmpFile ); 260 $tmpDir = "{$this->tmpParentDir}/$moduleName"; 261 $p->extractTo( $tmpDir ); 262 unset( $data, $p ); 263 264 if ( $info['dest'] === null ) { 265 // Default: Replace the entire directory 266 $toCopy = [ $tmpDir => $destDir ]; 267 } else { 268 // Expand and normalise the 'dest' entries 269 $toCopy = []; 270 foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { 271 // Use glob() to expand wildcards and check existence 272 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); 273 if ( !$fromPaths ) { 274 throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." ); 275 } 276 foreach ( $fromPaths as $fromPath ) { 277 $toCopy[$fromPath] = $toSubPath === null 278 ? "$destDir/" . basename( $fromPath ) 279 : "$destDir/$toSubPath/" . basename( $fromPath ); 280 } 281 } 282 } 283 foreach ( $toCopy as $from => $to ) { 284 if ( $this->action === 'verify' ) { 285 $this->verbose( "... verifying $to\n" ); 286 if ( is_dir( $from ) ) { 287 $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( 288 $from, 289 RecursiveDirectoryIterator::SKIP_DOTS 290 ) ); 291 /** @var SplFileInfo $file */ 292 foreach ( $rii as $file ) { 293 $remote = $file->getPathname(); 294 $local = strtr( $remote, [ $from => $to ] ); 295 if ( sha1_file( $remote ) !== sha1_file( $local ) ) { 296 $this->error( "File '$local' is different." ); 297 $this->hasErrors = true; 298 } 299 } 300 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { 301 $this->error( "File '$to' is different." ); 302 $this->hasErrors = true; 303 } 304 } elseif ( $this->action === 'update' ) { 305 $this->verbose( "... moving $from to $to\n" ); 306 wfMkdirParents( dirname( $to ) ); 307 if ( !rename( $from, $to ) ) { 308 throw new Exception( "Could not move $from to $to." ); 309 } 310 } 311 } 312 } 313 314 private function verbose( $text ) { 315 ( $this->verbosePrinter )( $text ); 316 } 317 318 private function output( $text ) { 319 ( $this->infoPrinter )( $text ); 320 } 321 322 private function error( $text ) { 323 ( $this->errorPrinter )( $text ); 324 } 325 326 private function cleanUp() { 327 wfRecursiveRemoveDir( $this->tmpParentDir ); 328 329 // Prune the cache of files we don't recognise. 330 $knownKeys = []; 331 foreach ( $this->registry as $info ) { 332 if ( $info['type'] === 'file' || $info['type'] === 'tar' ) { 333 $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] ); 334 } elseif ( $info['type'] === 'multi-file' ) { 335 foreach ( $info['files'] as $file ) { 336 $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] ); 337 } 338 } 339 } 340 foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) { 341 if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) { 342 unlink( $cacheFile ); 343 } 344 } 345 } 346 347 /** 348 * Basic YAML parser. 349 * 350 * Supports only string or object values, and 2 spaces indentation. 351 * 352 * @todo Just ship symfony/yaml. 353 * @param string $input 354 * @return array 355 */ 356 private function parseBasicYaml( $input ) { 357 $lines = explode( "\n", $input ); 358 $root = []; 359 $stack = [ &$root ]; 360 $prev = 0; 361 foreach ( $lines as $i => $text ) { 362 $line = $i + 1; 363 $trimmed = ltrim( $text, ' ' ); 364 if ( $trimmed === '' || $trimmed[0] === '#' ) { 365 continue; 366 } 367 $indent = strlen( $text ) - strlen( $trimmed ); 368 if ( $indent % 2 !== 0 ) { 369 throw new Exception( __METHOD__ . ": Odd indentation on line $line." ); 370 } 371 $depth = $indent === 0 ? 0 : ( $indent / 2 ); 372 if ( $depth < $prev ) { 373 // Close previous branches we can't re-enter 374 array_splice( $stack, $depth + 1 ); 375 } 376 if ( !array_key_exists( $depth, $stack ) ) { 377 throw new Exception( __METHOD__ . ": Too much indentation on line $line." ); 378 } 379 if ( strpos( $trimmed, ':' ) === false ) { 380 throw new Exception( __METHOD__ . ": Missing colon on line $line." ); 381 } 382 $dest =& $stack[ $depth ]; 383 if ( $dest === null ) { 384 // Promote from null to object 385 $dest = []; 386 } 387 list( $key, $val ) = explode( ':', $trimmed, 2 ); 388 $val = ltrim( $val, ' ' ); 389 if ( $val !== '' ) { 390 // Add string 391 $dest[ $key ] = $val; 392 } else { 393 // Add null (may become an object later) 394 $val = null; 395 $stack[] = &$val; 396 $dest[ $key ] = &$val; 397 } 398 $prev = $depth; 399 unset( $dest, $val ); 400 } 401 return $root; 402 } 403} 404