1<?php
2/**
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
17 *
18 * @file
19 * @ingroup Maintenance
20 */
21
22use MediaWiki\MediaWikiServices;
23use Wikimedia\AtEase\AtEase;
24
25/**
26 * Manage foreign resources registered with ResourceLoader.
27 *
28 * @since 1.32
29 */
30class ForeignResourceManager {
31	private $defaultAlgo = 'sha384';
32	private $hasErrors = false;
33	private $registryFile;
34	private $libDir;
35	private $tmpParentDir;
36	private $cacheDir;
37	/**
38	 * @var callable|Closure
39	 * @phan-var callable(string):void
40	 */
41	private $infoPrinter;
42	/**
43	 * @var callable|Closure
44	 * @phan-var callable(string):void
45	 */
46	private $errorPrinter;
47	/**
48	 * @var callable|Closure
49	 * @phan-var callable(string):void
50	 */
51	private $verbosePrinter;
52	private $action;
53	/** @var array[] */
54	private $registry;
55
56	/**
57	 * @param string $registryFile Path to YAML file
58	 * @param string $libDir Path to a modules directory
59	 * @param callable|null $infoPrinter Callback for printing info about the run.
60	 * @param callable|null $errorPrinter Callback for printing errors from the run.
61	 * @param callable|null $verbosePrinter Callback for printing extra verbose
62	 *  progress information from the run.
63	 */
64	public function __construct(
65		$registryFile,
66		$libDir,
67		callable $infoPrinter = null,
68		callable $errorPrinter = null,
69		callable $verbosePrinter = null
70	) {
71		$this->registryFile = $registryFile;
72		$this->libDir = $libDir;
73		$this->infoPrinter = $infoPrinter ?? function ( $_ ) {
74		};
75		$this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
76		$this->verbosePrinter = $verbosePrinter ?? function ( $_ ) {
77		};
78
79		// Use a temporary directory under the destination directory instead
80		// of wfTempDir() because PHP's rename() does not work across file
81		// systems, and the user's /tmp and $IP may be on different filesystems.
82		$this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
83
84		$cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
85		$this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
86	}
87
88	/**
89	 * @param string $action
90	 * @param string $module
91	 * @return bool
92	 * @throws Exception
93	 */
94	public function run( $action, $module ) {
95		$actions = [ 'update', 'verify', 'make-sri' ];
96		if ( !in_array( $action, $actions ) ) {
97			$this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
98			return false;
99		}
100		$this->action = $action;
101
102		$this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
103		if ( $module === 'all' ) {
104			$modules = $this->registry;
105		} elseif ( isset( $this->registry[ $module ] ) ) {
106			$modules = [ $module => $this->registry[ $module ] ];
107		} else {
108			$this->error( "Unknown module name.\n\nMust be one of:\n" .
109				wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
110				'.'
111			);
112			return false;
113		}
114
115		foreach ( $modules as $moduleName => $info ) {
116			$this->verbose( "\n### {$moduleName}\n\n" );
117			$destDir = "{$this->libDir}/$moduleName";
118
119			if ( $this->action === 'update' ) {
120				$this->output( "... updating '{$moduleName}'\n" );
121				$this->verbose( "... emptying directory for $moduleName\n" );
122				wfRecursiveRemoveDir( $destDir );
123			} elseif ( $this->action === 'verify' ) {
124				$this->output( "... verifying '{$moduleName}'\n" );
125			} else {
126				$this->output( "... checking '{$moduleName}'\n" );
127			}
128
129			$this->verbose( "... preparing {$this->tmpParentDir}\n" );
130			wfRecursiveRemoveDir( $this->tmpParentDir );
131			if ( !wfMkdirParents( $this->tmpParentDir ) ) {
132				throw new Exception( "Unable to create {$this->tmpParentDir}" );
133			}
134
135			if ( !isset( $info['type'] ) ) {
136				throw new Exception( "Module '$moduleName' must have a 'type' key." );
137			}
138			switch ( $info['type'] ) {
139				case 'tar':
140					$this->handleTypeTar( $moduleName, $destDir, $info );
141					break;
142				case 'file':
143					$this->handleTypeFile( $moduleName, $destDir, $info );
144					break;
145				case 'multi-file':
146					$this->handleTypeMultiFile( $moduleName, $destDir, $info );
147					break;
148				default:
149					throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
150			}
151		}
152
153		$this->output( "\nDone!\n" );
154		$this->cleanUp();
155		if ( $this->hasErrors ) {
156			// The verify mode should check all modules/files and fail after, not during.
157			return false;
158		}
159
160		return true;
161	}
162
163	private function cacheKey( $src, $integrity ) {
164		$key = basename( $src ) . '_' . substr( $integrity, -12 );
165		$key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
166		return rtrim( $key, '_' );
167	}
168
169	/**
170	 * @param string $key
171	 * @return string|false
172	 */
173	private function cacheGet( $key ) {
174		return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
175	}
176
177	private function cacheSet( $key, $data ) {
178		wfMkdirParents( $this->cacheDir );
179		file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
180	}
181
182	private function fetch( $src, $integrity ) {
183		$key = $this->cacheKey( $src, $integrity );
184		$data = $this->cacheGet( $key );
185		if ( $data ) {
186			return $data;
187		}
188
189		$req = MediaWikiServices::getInstance()->getHttpRequestFactory()
190			->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ );
191		if ( !$req->execute()->isOK() ) {
192			throw new Exception( "Failed to download resource at {$src}" );
193		}
194		if ( $req->getStatus() !== 200 ) {
195			throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
196		}
197		$data = $req->getContent();
198		$algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
199		$actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
200		if ( $integrity === $actualIntegrity ) {
201			$this->verbose( "... passed integrity check for {$src}\n" );
202			$this->cacheSet( $key, $data );
203		} elseif ( $this->action === 'make-sri' ) {
204			$this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
205		} else {
206			throw new Exception( "Integrity check failed for {$src}\n" .
207				"\tExpected: {$integrity}\n" .
208				"\tActual: {$actualIntegrity}"
209			);
210		}
211		return $data;
212	}
213
214	private function handleTypeFile( $moduleName, $destDir, array $info ) {
215		if ( !isset( $info['src'] ) ) {
216			throw new Exception( "Module '$moduleName' must have a 'src' key." );
217		}
218		$data = $this->fetch( $info['src'], $info['integrity'] ?? null );
219		$dest = $info['dest'] ?? basename( $info['src'] );
220		$path = "$destDir/$dest";
221		if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
222			throw new Exception( "File for '$moduleName' is different." );
223		}
224		if ( $this->action === 'update' ) {
225			wfMkdirParents( $destDir );
226			file_put_contents( "$destDir/$dest", $data );
227		}
228	}
229
230	private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
231		if ( !isset( $info['files'] ) ) {
232			throw new Exception( "Module '$moduleName' must have a 'files' key." );
233		}
234		foreach ( $info['files'] as $dest => $file ) {
235			if ( !isset( $file['src'] ) ) {
236				throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
237			}
238			$data = $this->fetch( $file['src'], $file['integrity'] ?? null );
239			$path = "$destDir/$dest";
240			if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
241				throw new Exception( "File '$dest' for '$moduleName' is different." );
242			} elseif ( $this->action === 'update' ) {
243				wfMkdirParents( $destDir );
244				file_put_contents( "$destDir/$dest", $data );
245			}
246		}
247	}
248
249	private function handleTypeTar( $moduleName, $destDir, array $info ) {
250		$info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
251		if ( $info['src'] === null ) {
252			throw new Exception( "Module '$moduleName' must have a 'src' key." );
253		}
254		// Download the resource to a temporary file and open it
255		$data = $this->fetch( $info['src'], $info['integrity' ] );
256		$tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
257		$this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
258		file_put_contents( $tmpFile, $data );
259		$p = new PharData( $tmpFile );
260		$tmpDir = "{$this->tmpParentDir}/$moduleName";
261		$p->extractTo( $tmpDir );
262		unset( $data, $p );
263
264		if ( $info['dest'] === null ) {
265			// Default: Replace the entire directory
266			$toCopy = [ $tmpDir => $destDir ];
267		} else {
268			// Expand and normalise the 'dest' entries
269			$toCopy = [];
270			foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
271				// Use glob() to expand wildcards and check existence
272				$fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
273				if ( !$fromPaths ) {
274					throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
275				}
276				foreach ( $fromPaths as $fromPath ) {
277					$toCopy[$fromPath] = $toSubPath === null
278						? "$destDir/" . basename( $fromPath )
279						: "$destDir/$toSubPath/" . basename( $fromPath );
280				}
281			}
282		}
283		foreach ( $toCopy as $from => $to ) {
284			if ( $this->action === 'verify' ) {
285				$this->verbose( "... verifying $to\n" );
286				if ( is_dir( $from ) ) {
287					$rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
288						$from,
289						RecursiveDirectoryIterator::SKIP_DOTS
290					) );
291					/** @var SplFileInfo $file */
292					foreach ( $rii as $file ) {
293						$remote = $file->getPathname();
294						$local = strtr( $remote, [ $from => $to ] );
295						if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
296							$this->error( "File '$local' is different." );
297							$this->hasErrors = true;
298						}
299					}
300				} elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
301					$this->error( "File '$to' is different." );
302					$this->hasErrors = true;
303				}
304			} elseif ( $this->action === 'update' ) {
305				$this->verbose( "... moving $from to $to\n" );
306				wfMkdirParents( dirname( $to ) );
307				if ( !rename( $from, $to ) ) {
308					throw new Exception( "Could not move $from to $to." );
309				}
310			}
311		}
312	}
313
314	private function verbose( $text ) {
315		( $this->verbosePrinter )( $text );
316	}
317
318	private function output( $text ) {
319		( $this->infoPrinter )( $text );
320	}
321
322	private function error( $text ) {
323		( $this->errorPrinter )( $text );
324	}
325
326	private function cleanUp() {
327		wfRecursiveRemoveDir( $this->tmpParentDir );
328
329		// Prune the cache of files we don't recognise.
330		$knownKeys = [];
331		foreach ( $this->registry as $info ) {
332			if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
333				$knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
334			} elseif ( $info['type'] === 'multi-file' ) {
335				foreach ( $info['files'] as $file ) {
336					$knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
337				}
338			}
339		}
340		foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
341			if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
342				unlink( $cacheFile );
343			}
344		}
345	}
346
347	/**
348	 * Basic YAML parser.
349	 *
350	 * Supports only string or object values, and 2 spaces indentation.
351	 *
352	 * @todo Just ship symfony/yaml.
353	 * @param string $input
354	 * @return array
355	 */
356	private function parseBasicYaml( $input ) {
357		$lines = explode( "\n", $input );
358		$root = [];
359		$stack = [ &$root ];
360		$prev = 0;
361		foreach ( $lines as $i => $text ) {
362			$line = $i + 1;
363			$trimmed = ltrim( $text, ' ' );
364			if ( $trimmed === '' || $trimmed[0] === '#' ) {
365				continue;
366			}
367			$indent = strlen( $text ) - strlen( $trimmed );
368			if ( $indent % 2 !== 0 ) {
369				throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
370			}
371			$depth = $indent === 0 ? 0 : ( $indent / 2 );
372			if ( $depth < $prev ) {
373				// Close previous branches we can't re-enter
374				array_splice( $stack, $depth + 1 );
375			}
376			if ( !array_key_exists( $depth, $stack ) ) {
377				throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
378			}
379			if ( strpos( $trimmed, ':' ) === false ) {
380				throw new Exception( __METHOD__ . ": Missing colon on line $line." );
381			}
382			$dest =& $stack[ $depth ];
383			if ( $dest === null ) {
384				// Promote from null to object
385				$dest = [];
386			}
387			list( $key, $val ) = explode( ':', $trimmed, 2 );
388			$val = ltrim( $val, ' ' );
389			if ( $val !== '' ) {
390				// Add string
391				$dest[ $key ] = $val;
392			} else {
393				// Add null (may become an object later)
394				$val = null;
395				$stack[] = &$val;
396				$dest[ $key ] = &$val;
397			}
398			$prev = $depth;
399			unset( $dest, $val );
400		}
401		return $root;
402	}
403}
404