1<?php 2/** 3 * Cache of the contents of localisation files. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program; if not, write to the Free Software Foundation, Inc., 17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 * http://www.gnu.org/copyleft/gpl.html 19 * 20 * @file 21 */ 22 23use CLDRPluralRuleParser\Error as CLDRPluralRuleError; 24use CLDRPluralRuleParser\Evaluator; 25use MediaWiki\Config\ServiceOptions; 26use MediaWiki\HookContainer\HookContainer; 27use MediaWiki\HookContainer\HookRunner; 28use MediaWiki\Languages\LanguageNameUtils; 29use Psr\Log\LoggerInterface; 30 31/** 32 * Class for caching the contents of localisation files, Messages*.php 33 * and *.i18n.php. 34 * 35 * An instance of this class is available using MediaWikiServices. 36 * 37 * The values retrieved from here are merged, containing items from extension 38 * files, core messages files and the language fallback sequence (e.g. zh-cn -> 39 * zh-hans -> en ). Some common errors are corrected, for example namespace 40 * names with spaces instead of underscores, but heavyweight processing, such 41 * as grammatical transformation, is done by the caller. 42 */ 43class LocalisationCache { 44 public const VERSION = 4; 45 46 /** @var ServiceOptions */ 47 private $options; 48 49 /** 50 * True if recaching should only be done on an explicit call to recache(). 51 * Setting this reduces the overhead of cache freshness checking, which 52 * requires doing a stat() for every extension i18n file. 53 */ 54 private $manualRecache = false; 55 56 /** 57 * The cache data. 3-d array, where the first key is the language code, 58 * the second key is the item key e.g. 'messages', and the third key is 59 * an item specific subkey index. Some items are not arrays and so for those 60 * items, there are no subkeys. 61 */ 62 protected $data = []; 63 64 /** 65 * The persistent store object. An instance of LCStore. 66 * 67 * @var LCStore 68 */ 69 private $store; 70 71 /** 72 * @var LoggerInterface 73 */ 74 private $logger; 75 76 /** @var HookRunner */ 77 private $hookRunner; 78 79 /** @var callable[] See comment for parameter in constructor */ 80 private $clearStoreCallbacks; 81 82 /** @var LanguageNameUtils */ 83 private $langNameUtils; 84 85 /** 86 * A 2-d associative array, code/key, where presence indicates that the item 87 * is loaded. Value arbitrary. 88 * 89 * For split items, if set, this indicates that all of the subitems have been 90 * loaded. 91 * 92 */ 93 private $loadedItems = []; 94 95 /** 96 * A 3-d associative array, code/key/subkey, where presence indicates that 97 * the subitem is loaded. Only used for the split items, i.e. messages. 98 */ 99 private $loadedSubitems = []; 100 101 /** 102 * An array where presence of a key indicates that that language has been 103 * initialised. Initialisation includes checking for cache expiry and doing 104 * any necessary updates. 105 */ 106 private $initialisedLangs = []; 107 108 /** 109 * An array mapping non-existent pseudo-languages to fallback languages. This 110 * is filled by initShallowFallback() when data is requested from a language 111 * that lacks a Messages*.php file. 112 */ 113 private $shallowFallbacks = []; 114 115 /** 116 * An array where the keys are codes that have been recached by this instance. 117 */ 118 private $recachedLangs = []; 119 120 /** 121 * All item keys 122 */ 123 public static $allKeys = [ 124 'fallback', 'namespaceNames', 'bookstoreList', 125 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 126 'digitTransformTable', 'separatorTransformTable', 127 'minimumGroupingDigits', 'fallback8bitEncoding', 128 'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset', 129 'namespaceAliases', 'dateFormats', 'datePreferences', 130 'datePreferenceMigrationMap', 'defaultDateFormat', 131 'specialPageAliases', 'imageFiles', 'preloadedMessages', 132 'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules', 133 'pluralRuleTypes', 'compiledPluralRules', 134 ]; 135 136 /** 137 * Keys for items which consist of associative arrays, which may be merged 138 * by a fallback sequence. 139 */ 140 public static $mergeableMapKeys = [ 'messages', 'namespaceNames', 141 'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages' 142 ]; 143 144 /** 145 * Keys for items which are a numbered array. 146 */ 147 public static $mergeableListKeys = []; 148 149 /** 150 * Keys for items which contain an array of arrays of equivalent aliases 151 * for each subitem. The aliases may be merged by a fallback sequence. 152 */ 153 public static $mergeableAliasListKeys = [ 'specialPageAliases' ]; 154 155 /** 156 * Keys for items which contain an associative array, and may be merged if 157 * the primary value contains the special array key "inherit". That array 158 * key is removed after the first merge. 159 */ 160 public static $optionalMergeKeys = [ 'bookstoreList' ]; 161 162 /** 163 * Keys for items that are formatted like $magicWords 164 */ 165 public static $magicWordKeys = [ 'magicWords' ]; 166 167 /** 168 * Keys for items where the subitems are stored in the backend separately. 169 */ 170 public static $splitKeys = [ 'messages' ]; 171 172 /** 173 * Keys which are loaded automatically by initLanguage() 174 */ 175 public static $preloadedKeys = [ 'dateFormats', 'namespaceNames' ]; 176 177 /** 178 * Associative array of cached plural rules. The key is the language code, 179 * the value is an array of plural rules for that language. 180 */ 181 private $pluralRules = null; 182 183 /** 184 * Associative array of cached plural rule types. The key is the language 185 * code, the value is an array of plural rule types for that language. For 186 * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many']. 187 * The index for each rule type matches the index for the rule in 188 * $pluralRules, thus allowing correlation between the two. The reason we 189 * don't just use the type names as the keys in $pluralRules is because 190 * Language::convertPlural applies the rules based on numeric order (or 191 * explicit numeric parameter), not based on the name of the rule type. For 192 * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than 193 * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}. 194 */ 195 private $pluralRuleTypes = null; 196 197 private $mergeableKeys = null; 198 199 /** 200 * Return a suitable LCStore as specified by the given configuration. 201 * 202 * @since 1.34 203 * @param array $conf In the format of $wgLocalisationCacheConf 204 * @param string|false|null $fallbackCacheDir In case 'storeDirectory' isn't specified 205 * @return LCStore 206 */ 207 public static function getStoreFromConf( array $conf, $fallbackCacheDir ) : LCStore { 208 $storeArg = []; 209 $storeArg['directory'] = 210 $conf['storeDirectory'] ?: $fallbackCacheDir; 211 212 if ( !empty( $conf['storeClass'] ) ) { 213 $storeClass = $conf['storeClass']; 214 } elseif ( $conf['store'] === 'files' || $conf['store'] === 'file' || 215 ( $conf['store'] === 'detect' && $storeArg['directory'] ) 216 ) { 217 $storeClass = LCStoreCDB::class; 218 } elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) { 219 $storeClass = LCStoreDB::class; 220 $storeArg['server'] = $conf['storeServer'] ?? []; 221 } elseif ( $conf['store'] === 'array' ) { 222 $storeClass = LCStoreStaticArray::class; 223 } else { 224 throw new MWException( 225 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.' 226 ); 227 } 228 229 return new $storeClass( $storeArg ); 230 } 231 232 /** 233 * @internal For use by ServiceWiring 234 */ 235 public const CONSTRUCTOR_OPTIONS = [ 236 // True to treat all files as expired until they are regenerated by this object. 237 'forceRecache', 238 'manualRecache', 239 'ExtensionMessagesFiles', 240 'MessagesDirs', 241 ]; 242 243 /** 244 * For constructor parameters, see the documentation in DefaultSettings.php 245 * for $wgLocalisationCacheConf. 246 * 247 * Do not construct this directly. Use MediaWikiServices. 248 * 249 * @param ServiceOptions $options 250 * @param LCStore $store What backend to use for storage 251 * @param LoggerInterface $logger 252 * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be 253 * used to clear other caches that depend on this one, such as ResourceLoader's 254 * MessageBlobStore. 255 * @param LanguageNameUtils $langNameUtils 256 * @param HookContainer $hookContainer 257 * @throws MWException 258 */ 259 public function __construct( 260 ServiceOptions $options, 261 LCStore $store, 262 LoggerInterface $logger, 263 array $clearStoreCallbacks, 264 LanguageNameUtils $langNameUtils, 265 HookContainer $hookContainer 266 ) { 267 $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS ); 268 269 $this->options = $options; 270 $this->store = $store; 271 $this->logger = $logger; 272 $this->clearStoreCallbacks = $clearStoreCallbacks; 273 $this->langNameUtils = $langNameUtils; 274 $this->hookRunner = new HookRunner( $hookContainer ); 275 276 // Keep this separate from $this->options so it can be mutable 277 $this->manualRecache = $options->get( 'manualRecache' ); 278 } 279 280 /** 281 * Returns true if the given key is mergeable, that is, if it is an associative 282 * array which can be merged through a fallback sequence. 283 * @param string $key 284 * @return bool 285 */ 286 public function isMergeableKey( $key ) { 287 if ( $this->mergeableKeys === null ) { 288 $this->mergeableKeys = array_flip( array_merge( 289 self::$mergeableMapKeys, 290 self::$mergeableListKeys, 291 self::$mergeableAliasListKeys, 292 self::$optionalMergeKeys, 293 self::$magicWordKeys 294 ) ); 295 } 296 297 return isset( $this->mergeableKeys[$key] ); 298 } 299 300 /** 301 * Get a cache item. 302 * 303 * Warning: this may be slow for split items (messages), since it will 304 * need to fetch all of the subitems from the cache individually. 305 * @param string $code 306 * @param string $key 307 * @return mixed 308 */ 309 public function getItem( $code, $key ) { 310 if ( !isset( $this->loadedItems[$code][$key] ) ) { 311 $this->loadItem( $code, $key ); 312 } 313 314 if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) { 315 return $this->shallowFallbacks[$code]; 316 } 317 318 return $this->data[$code][$key]; 319 } 320 321 /** 322 * Get a subitem, for instance a single message for a given language. 323 * @param string $code 324 * @param string $key 325 * @param string $subkey 326 * @return mixed|null 327 */ 328 public function getSubitem( $code, $key, $subkey ) { 329 if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) && 330 !isset( $this->loadedItems[$code][$key] ) 331 ) { 332 $this->loadSubitem( $code, $key, $subkey ); 333 } 334 335 return $this->data[$code][$key][$subkey] ?? null; 336 } 337 338 /** 339 * Get the list of subitem keys for a given item. 340 * 341 * This is faster than array_keys($lc->getItem(...)) for the items listed in 342 * self::$splitKeys. 343 * 344 * Will return null if the item is not found, or false if the item is not an 345 * array. 346 * @param string $code 347 * @param string $key 348 * @return bool|null|string|string[] 349 */ 350 public function getSubitemList( $code, $key ) { 351 if ( in_array( $key, self::$splitKeys ) ) { 352 return $this->getSubitem( $code, 'list', $key ); 353 } else { 354 $item = $this->getItem( $code, $key ); 355 if ( is_array( $item ) ) { 356 return array_keys( $item ); 357 } else { 358 return false; 359 } 360 } 361 } 362 363 /** 364 * Load an item into the cache. 365 * @param string $code 366 * @param string $key 367 */ 368 protected function loadItem( $code, $key ) { 369 if ( !isset( $this->initialisedLangs[$code] ) ) { 370 $this->initLanguage( $code ); 371 } 372 373 // Check to see if initLanguage() loaded it for us 374 if ( isset( $this->loadedItems[$code][$key] ) ) { 375 return; 376 } 377 378 if ( isset( $this->shallowFallbacks[$code] ) ) { 379 $this->loadItem( $this->shallowFallbacks[$code], $key ); 380 381 return; 382 } 383 384 if ( in_array( $key, self::$splitKeys ) ) { 385 $subkeyList = $this->getSubitem( $code, 'list', $key ); 386 foreach ( $subkeyList as $subkey ) { 387 if ( isset( $this->data[$code][$key][$subkey] ) ) { 388 continue; 389 } 390 $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey ); 391 } 392 } else { 393 $this->data[$code][$key] = $this->store->get( $code, $key ); 394 } 395 396 $this->loadedItems[$code][$key] = true; 397 } 398 399 /** 400 * Load a subitem into the cache 401 * @param string $code 402 * @param string $key 403 * @param string $subkey 404 */ 405 protected function loadSubitem( $code, $key, $subkey ) { 406 if ( !in_array( $key, self::$splitKeys ) ) { 407 $this->loadItem( $code, $key ); 408 409 return; 410 } 411 412 if ( !isset( $this->initialisedLangs[$code] ) ) { 413 $this->initLanguage( $code ); 414 } 415 416 // Check to see if initLanguage() loaded it for us 417 if ( isset( $this->loadedItems[$code][$key] ) || 418 isset( $this->loadedSubitems[$code][$key][$subkey] ) 419 ) { 420 return; 421 } 422 423 if ( isset( $this->shallowFallbacks[$code] ) ) { 424 $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey ); 425 426 return; 427 } 428 429 $value = $this->store->get( $code, "$key:$subkey" ); 430 $this->data[$code][$key][$subkey] = $value; 431 $this->loadedSubitems[$code][$key][$subkey] = true; 432 } 433 434 /** 435 * Returns true if the cache identified by $code is missing or expired. 436 * 437 * @param string $code 438 * 439 * @return bool 440 */ 441 public function isExpired( $code ) { 442 if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) { 443 $this->logger->debug( __METHOD__ . "($code): forced reload" ); 444 445 return true; 446 } 447 448 $deps = $this->store->get( $code, 'deps' ); 449 $keys = $this->store->get( $code, 'list' ); 450 $preload = $this->store->get( $code, 'preload' ); 451 // Different keys may expire separately for some stores 452 if ( $deps === null || $keys === null || $preload === null ) { 453 $this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" ); 454 455 return true; 456 } 457 458 foreach ( $deps as $dep ) { 459 // Because we're unserializing stuff from cache, we 460 // could receive objects of classes that don't exist 461 // anymore (e.g. uninstalled extensions) 462 // When this happens, always expire the cache 463 if ( !$dep instanceof CacheDependency || $dep->isExpired() ) { 464 $this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " . 465 get_class( $dep ) ); 466 467 return true; 468 } 469 } 470 471 return false; 472 } 473 474 /** 475 * Initialise a language in this object. Rebuild the cache if necessary. 476 * @param string $code 477 * @throws MWException 478 */ 479 protected function initLanguage( $code ) { 480 if ( isset( $this->initialisedLangs[$code] ) ) { 481 return; 482 } 483 484 $this->initialisedLangs[$code] = true; 485 486 # If the code is of the wrong form for a Messages*.php file, do a shallow fallback 487 if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) { 488 $this->initShallowFallback( $code, 'en' ); 489 490 return; 491 } 492 493 # Recache the data if necessary 494 if ( !$this->manualRecache && $this->isExpired( $code ) ) { 495 if ( $this->langNameUtils->isSupportedLanguage( $code ) ) { 496 $this->recache( $code ); 497 } elseif ( $code === 'en' ) { 498 throw new MWException( 'MessagesEn.php is missing.' ); 499 } else { 500 $this->initShallowFallback( $code, 'en' ); 501 } 502 503 return; 504 } 505 506 # Preload some stuff 507 $preload = $this->getItem( $code, 'preload' ); 508 if ( $preload === null ) { 509 if ( $this->manualRecache ) { 510 // No Messages*.php file. Do shallow fallback to en. 511 if ( $code === 'en' ) { 512 throw new MWException( 'No localisation cache found for English. ' . 513 'Please run maintenance/rebuildLocalisationCache.php.' ); 514 } 515 $this->initShallowFallback( $code, 'en' ); 516 517 return; 518 } else { 519 throw new MWException( 'Invalid or missing localisation cache.' ); 520 } 521 } 522 $this->data[$code] = $preload; 523 foreach ( $preload as $key => $item ) { 524 if ( in_array( $key, self::$splitKeys ) ) { 525 foreach ( $item as $subkey => $subitem ) { 526 $this->loadedSubitems[$code][$key][$subkey] = true; 527 } 528 } else { 529 $this->loadedItems[$code][$key] = true; 530 } 531 } 532 } 533 534 /** 535 * Create a fallback from one language to another, without creating a 536 * complete persistent cache. 537 * @param string $primaryCode 538 * @param string $fallbackCode 539 */ 540 public function initShallowFallback( $primaryCode, $fallbackCode ) { 541 $this->data[$primaryCode] =& $this->data[$fallbackCode]; 542 $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode]; 543 $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode]; 544 $this->shallowFallbacks[$primaryCode] = $fallbackCode; 545 } 546 547 /** 548 * Read a PHP file containing localisation data. 549 * @param string $_fileName 550 * @param string $_fileType 551 * @throws MWException 552 * @return array 553 */ 554 protected function readPHPFile( $_fileName, $_fileType ) { 555 include $_fileName; 556 557 $data = []; 558 if ( $_fileType == 'core' || $_fileType == 'extension' ) { 559 foreach ( self::$allKeys as $key ) { 560 // Not all keys are set in language files, so 561 // check they exist first 562 if ( isset( $$key ) ) { 563 $data[$key] = $$key; 564 } 565 } 566 } elseif ( $_fileType == 'aliases' ) { 567 // @phan-suppress-next-line PhanImpossibleCondition May be set in included file 568 if ( isset( $aliases ) ) { 569 $data['aliases'] = $aliases; 570 } 571 } else { 572 throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" ); 573 } 574 575 return $data; 576 } 577 578 /** 579 * Read a JSON file containing localisation messages. 580 * @param string $fileName Name of file to read 581 * @throws MWException If there is a syntax error in the JSON file 582 * @return array Array with a 'messages' key, or empty array if the file doesn't exist 583 */ 584 public function readJSONFile( $fileName ) { 585 if ( !is_readable( $fileName ) ) { 586 return []; 587 } 588 589 $json = file_get_contents( $fileName ); 590 if ( $json === false ) { 591 return []; 592 } 593 594 $data = FormatJson::decode( $json, true ); 595 if ( $data === null ) { 596 throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" ); 597 } 598 599 // Remove keys starting with '@', they're reserved for metadata and non-message data 600 foreach ( $data as $key => $unused ) { 601 if ( $key === '' || $key[0] === '@' ) { 602 unset( $data[$key] ); 603 } 604 } 605 606 // The JSON format only supports messages, none of the other variables, so wrap the data 607 return [ 'messages' => $data ]; 608 } 609 610 /** 611 * Get the compiled plural rules for a given language from the XML files. 612 * @since 1.20 613 * @param string $code 614 * @return array|null 615 */ 616 public function getCompiledPluralRules( $code ) { 617 $rules = $this->getPluralRules( $code ); 618 if ( $rules === null ) { 619 return null; 620 } 621 try { 622 $compiledRules = Evaluator::compile( $rules ); 623 } catch ( CLDRPluralRuleError $e ) { 624 $this->logger->debug( $e->getMessage() ); 625 626 return []; 627 } 628 629 return $compiledRules; 630 } 631 632 /** 633 * Get the plural rules for a given language from the XML files. 634 * Cached. 635 * @since 1.20 636 * @param string $code 637 * @return array|null 638 */ 639 public function getPluralRules( $code ) { 640 if ( $this->pluralRules === null ) { 641 $this->loadPluralFiles(); 642 } 643 return $this->pluralRules[$code] ?? null; 644 } 645 646 /** 647 * Get the plural rule types for a given language from the XML files. 648 * Cached. 649 * @since 1.22 650 * @param string $code 651 * @return array|null 652 */ 653 public function getPluralRuleTypes( $code ) { 654 if ( $this->pluralRuleTypes === null ) { 655 $this->loadPluralFiles(); 656 } 657 return $this->pluralRuleTypes[$code] ?? null; 658 } 659 660 /** 661 * Load the plural XML files. 662 */ 663 protected function loadPluralFiles() { 664 global $IP; 665 $cldrPlural = "$IP/languages/data/plurals.xml"; 666 $mwPlural = "$IP/languages/data/plurals-mediawiki.xml"; 667 // Load CLDR plural rules 668 $this->loadPluralFile( $cldrPlural ); 669 if ( file_exists( $mwPlural ) ) { 670 // Override or extend 671 $this->loadPluralFile( $mwPlural ); 672 } 673 } 674 675 /** 676 * Load a plural XML file with the given filename, compile the relevant 677 * rules, and save the compiled rules in a process-local cache. 678 * 679 * @param string $fileName 680 * @throws MWException 681 */ 682 protected function loadPluralFile( $fileName ) { 683 // Use file_get_contents instead of DOMDocument::load (T58439) 684 $xml = file_get_contents( $fileName ); 685 if ( !$xml ) { 686 throw new MWException( "Unable to read plurals file $fileName" ); 687 } 688 $doc = new DOMDocument; 689 $doc->loadXML( $xml ); 690 $rulesets = $doc->getElementsByTagName( "pluralRules" ); 691 foreach ( $rulesets as $ruleset ) { 692 $codes = $ruleset->getAttribute( 'locales' ); 693 $rules = []; 694 $ruleTypes = []; 695 $ruleElements = $ruleset->getElementsByTagName( "pluralRule" ); 696 foreach ( $ruleElements as $elt ) { 697 $ruleType = $elt->getAttribute( 'count' ); 698 if ( $ruleType === 'other' ) { 699 // Don't record "other" rules, which have an empty condition 700 continue; 701 } 702 $rules[] = $elt->nodeValue; 703 $ruleTypes[] = $ruleType; 704 } 705 foreach ( explode( ' ', $codes ) as $code ) { 706 $this->pluralRules[$code] = $rules; 707 $this->pluralRuleTypes[$code] = $ruleTypes; 708 } 709 } 710 } 711 712 /** 713 * Read the data from the source files for a given language, and register 714 * the relevant dependencies in the $deps array. If the localisation 715 * exists, the data array is returned, otherwise false is returned. 716 * 717 * @param string $code 718 * @param array &$deps 719 * @return array 720 */ 721 protected function readSourceFilesAndRegisterDeps( $code, &$deps ) { 722 global $IP; 723 724 // This reads in the PHP i18n file with non-messages l10n data 725 $fileName = $this->langNameUtils->getMessagesFileName( $code ); 726 if ( !file_exists( $fileName ) ) { 727 $data = []; 728 } else { 729 $deps[] = new FileDependency( $fileName ); 730 $data = $this->readPHPFile( $fileName, 'core' ); 731 } 732 733 # Load CLDR plural rules for JavaScript 734 $data['pluralRules'] = $this->getPluralRules( $code ); 735 # And for PHP 736 $data['compiledPluralRules'] = $this->getCompiledPluralRules( $code ); 737 # Load plural rule types 738 $data['pluralRuleTypes'] = $this->getPluralRuleTypes( $code ); 739 740 $deps['plurals'] = new FileDependency( "$IP/languages/data/plurals.xml" ); 741 $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" ); 742 743 return $data; 744 } 745 746 /** 747 * Merge two localisation values, a primary and a fallback, overwriting the 748 * primary value in place. 749 * @param string $key 750 * @param mixed &$value 751 * @param mixed $fallbackValue 752 */ 753 protected function mergeItem( $key, &$value, $fallbackValue ) { 754 if ( $value !== null ) { 755 if ( $fallbackValue !== null ) { 756 if ( in_array( $key, self::$mergeableMapKeys ) ) { 757 $value += $fallbackValue; 758 } elseif ( in_array( $key, self::$mergeableListKeys ) ) { 759 $value = array_unique( array_merge( $fallbackValue, $value ) ); 760 } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) { 761 $value = array_merge_recursive( $value, $fallbackValue ); 762 } elseif ( in_array( $key, self::$optionalMergeKeys ) ) { 763 if ( !empty( $value['inherit'] ) ) { 764 $value = array_merge( $fallbackValue, $value ); 765 } 766 767 unset( $value['inherit'] ); 768 } elseif ( in_array( $key, self::$magicWordKeys ) ) { 769 $this->mergeMagicWords( $value, $fallbackValue ); 770 } 771 } 772 } else { 773 $value = $fallbackValue; 774 } 775 } 776 777 /** 778 * @param mixed &$value 779 * @param mixed $fallbackValue 780 */ 781 protected function mergeMagicWords( &$value, $fallbackValue ) { 782 foreach ( $fallbackValue as $magicName => $fallbackInfo ) { 783 if ( !isset( $value[$magicName] ) ) { 784 $value[$magicName] = $fallbackInfo; 785 } else { 786 $oldSynonyms = array_slice( $fallbackInfo, 1 ); 787 $newSynonyms = array_slice( $value[$magicName], 1 ); 788 $synonyms = array_values( array_unique( array_merge( 789 $newSynonyms, $oldSynonyms ) ) ); 790 $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms ); 791 } 792 } 793 } 794 795 /** 796 * Given an array mapping language code to localisation value, such as is 797 * found in extension *.i18n.php files, iterate through a fallback sequence 798 * to merge the given data with an existing primary value. 799 * 800 * Returns true if any data from the extension array was used, false 801 * otherwise. 802 * @param array $codeSequence 803 * @param string $key 804 * @param mixed &$value 805 * @param mixed $fallbackValue 806 * @return bool 807 */ 808 protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) { 809 $used = false; 810 foreach ( $codeSequence as $code ) { 811 if ( isset( $fallbackValue[$code] ) ) { 812 $this->mergeItem( $key, $value, $fallbackValue[$code] ); 813 $used = true; 814 } 815 } 816 817 return $used; 818 } 819 820 /** 821 * Gets the combined list of messages dirs from 822 * core and extensions 823 * 824 * @since 1.25 825 * @return array 826 */ 827 public function getMessagesDirs() { 828 global $IP; 829 830 return [ 831 'core' => "$IP/languages/i18n", 832 'exif' => "$IP/languages/i18n/exif", 833 'api' => "$IP/includes/api/i18n", 834 'rest' => "$IP/includes/Rest/i18n", 835 'oojs-ui' => "$IP/resources/lib/ooui/i18n", 836 'paramvalidator' => "$IP/includes/libs/ParamValidator/i18n", 837 ] + $this->options->get( 'MessagesDirs' ); 838 } 839 840 /** 841 * Load localisation data for a given language for both core and extensions 842 * and save it to the persistent cache store and the process cache 843 * @param string $code 844 * @throws MWException 845 */ 846 public function recache( $code ) { 847 if ( !$code ) { 848 throw new MWException( "Invalid language code requested" ); 849 } 850 $this->recachedLangs[ $code ] = true; 851 852 # Initial values 853 $initialData = array_fill_keys( self::$allKeys, null ); 854 $coreData = $initialData; 855 $deps = []; 856 857 # Load the primary localisation from the source file 858 $data = $this->readSourceFilesAndRegisterDeps( $code, $deps ); 859 $this->logger->debug( __METHOD__ . ": got localisation for $code from source" ); 860 861 # Merge primary localisation 862 foreach ( $data as $key => $value ) { 863 $this->mergeItem( $key, $coreData[ $key ], $value ); 864 } 865 866 # Fill in the fallback if it's not there already 867 // @phan-suppress-next-line PhanSuspiciousValueComparison 868 if ( ( $coreData['fallback'] === null || $coreData['fallback'] === false ) && $code === 'en' ) { 869 $coreData['fallback'] = false; 870 $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = []; 871 } else { 872 if ( $coreData['fallback'] !== null ) { 873 $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) ); 874 } else { 875 $coreData['fallbackSequence'] = []; 876 } 877 $len = count( $coreData['fallbackSequence'] ); 878 879 # Before we add the 'en' fallback for messages, keep a copy of 880 # the original fallback sequence 881 $coreData['originalFallbackSequence'] = $coreData['fallbackSequence']; 882 883 # Ensure that the sequence ends at 'en' for messages 884 if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) { 885 $coreData['fallbackSequence'][] = 'en'; 886 } 887 } 888 889 $codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] ); 890 $messageDirs = $this->getMessagesDirs(); 891 892 # Load non-JSON localisation data for extensions 893 $extensionData = array_fill_keys( $codeSequence, $initialData ); 894 foreach ( $this->options->get( 'ExtensionMessagesFiles' ) as $extension => $fileName ) { 895 if ( isset( $messageDirs[$extension] ) ) { 896 # This extension has JSON message data; skip the PHP shim 897 continue; 898 } 899 900 $data = $this->readPHPFile( $fileName, 'extension' ); 901 $used = false; 902 903 foreach ( $data as $key => $item ) { 904 foreach ( $codeSequence as $csCode ) { 905 if ( isset( $item[$csCode] ) ) { 906 $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] ); 907 $used = true; 908 } 909 } 910 } 911 912 if ( $used ) { 913 $deps[] = new FileDependency( $fileName ); 914 } 915 } 916 917 # Load the localisation data for each fallback, then merge it into the full array 918 $allData = $initialData; 919 foreach ( $codeSequence as $csCode ) { 920 $csData = $initialData; 921 922 # Load core messages and the extension localisations. 923 foreach ( $messageDirs as $dirs ) { 924 foreach ( (array)$dirs as $dir ) { 925 $fileName = "$dir/$csCode.json"; 926 $data = $this->readJSONFile( $fileName ); 927 928 foreach ( $data as $key => $item ) { 929 $this->mergeItem( $key, $csData[$key], $item ); 930 } 931 932 $deps[] = new FileDependency( $fileName ); 933 } 934 } 935 936 # Merge non-JSON extension data 937 if ( isset( $extensionData[$csCode] ) ) { 938 foreach ( $extensionData[$csCode] as $key => $item ) { 939 $this->mergeItem( $key, $csData[$key], $item ); 940 } 941 } 942 943 if ( $csCode === $code ) { 944 # Merge core data into extension data 945 foreach ( $coreData as $key => $item ) { 946 $this->mergeItem( $key, $csData[$key], $item ); 947 } 948 } else { 949 # Load the secondary localisation from the source file to 950 # avoid infinite cycles on cyclic fallbacks 951 $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps ); 952 # Only merge the keys that make sense to merge 953 foreach ( self::$allKeys as $key ) { 954 if ( !isset( $fbData[ $key ] ) ) { 955 continue; 956 } 957 958 if ( ( $coreData[ $key ] ) === null || $this->isMergeableKey( $key ) ) { 959 $this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] ); 960 } 961 } 962 } 963 964 # Allow extensions an opportunity to adjust the data for this 965 # fallback 966 $this->hookRunner->onLocalisationCacheRecacheFallback( $this, $csCode, $csData ); 967 968 # Merge the data for this fallback into the final array 969 if ( $csCode === $code ) { 970 $allData = $csData; 971 } else { 972 foreach ( self::$allKeys as $key ) { 973 if ( !isset( $csData[$key] ) ) { 974 continue; 975 } 976 977 // @phan-suppress-next-line PhanTypeArraySuspiciousNullable 978 if ( $allData[$key] === null || $this->isMergeableKey( $key ) ) { 979 $this->mergeItem( $key, $allData[$key], $csData[$key] ); 980 } 981 } 982 } 983 } 984 985 # Add cache dependencies for any referenced globals 986 $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' ); 987 // The 'MessagesDirs' config setting is used in LocalisationCache::getMessagesDirs(). 988 // We use the key 'wgMessagesDirs' for historical reasons. 989 $deps['wgMessagesDirs'] = new MainConfigDependency( 'MessagesDirs' ); 990 $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' ); 991 992 # Add dependencies to the cache entry 993 $allData['deps'] = $deps; 994 995 # Replace spaces with underscores in namespace names 996 $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] ); 997 998 # And do the same for special page aliases. $page is an array. 999 foreach ( $allData['specialPageAliases'] as &$page ) { 1000 $page = str_replace( ' ', '_', $page ); 1001 } 1002 # Decouple the reference to prevent accidental damage 1003 unset( $page ); 1004 1005 # If there were no plural rules, return an empty array 1006 if ( $allData['pluralRules'] === null ) { 1007 $allData['pluralRules'] = []; 1008 } 1009 if ( $allData['compiledPluralRules'] === null ) { 1010 $allData['compiledPluralRules'] = []; 1011 } 1012 # If there were no plural rule types, return an empty array 1013 if ( $allData['pluralRuleTypes'] === null ) { 1014 $allData['pluralRuleTypes'] = []; 1015 } 1016 1017 # Set the list keys 1018 $allData['list'] = []; 1019 foreach ( self::$splitKeys as $key ) { 1020 $allData['list'][$key] = array_keys( $allData[$key] ); 1021 } 1022 # Run hooks 1023 $unused = true; // Used to be $purgeBlobs, removed in 1.34 1024 $this->hookRunner->onLocalisationCacheRecache( $this, $code, $allData, $unused ); 1025 1026 if ( $allData['namespaceNames'] === null ) { 1027 throw new MWException( __METHOD__ . ': Localisation data failed sanity check! ' . 1028 'Check that your languages/messages/MessagesEn.php file is intact.' ); 1029 } 1030 1031 # Set the preload key 1032 $allData['preload'] = $this->buildPreload( $allData ); 1033 1034 # Save to the process cache and register the items loaded 1035 $this->data[$code] = $allData; 1036 foreach ( $allData as $key => $item ) { 1037 $this->loadedItems[$code][$key] = true; 1038 } 1039 1040 # Save to the persistent cache 1041 $this->store->startWrite( $code ); 1042 foreach ( $allData as $key => $value ) { 1043 if ( in_array( $key, self::$splitKeys ) ) { 1044 foreach ( $value as $subkey => $subvalue ) { 1045 $this->store->set( "$key:$subkey", $subvalue ); 1046 } 1047 } else { 1048 $this->store->set( $key, $value ); 1049 } 1050 } 1051 $this->store->finishWrite(); 1052 1053 # Clear out the MessageBlobStore 1054 # HACK: If using a null (i.e. disabled) storage backend, we 1055 # can't write to the MessageBlobStore either 1056 if ( !$this->store instanceof LCStoreNull ) { 1057 foreach ( $this->clearStoreCallbacks as $callback ) { 1058 $callback(); 1059 } 1060 } 1061 } 1062 1063 /** 1064 * Build the preload item from the given pre-cache data. 1065 * 1066 * The preload item will be loaded automatically, improving performance 1067 * for the commonly-requested items it contains. 1068 * @param array $data 1069 * @return array 1070 */ 1071 protected function buildPreload( $data ) { 1072 $preload = [ 'messages' => [] ]; 1073 foreach ( self::$preloadedKeys as $key ) { 1074 $preload[$key] = $data[$key]; 1075 } 1076 1077 foreach ( $data['preloadedMessages'] as $subkey ) { 1078 $subitem = $data['messages'][$subkey] ?? null; 1079 $preload['messages'][$subkey] = $subitem; 1080 } 1081 1082 return $preload; 1083 } 1084 1085 /** 1086 * Unload the data for a given language from the object cache. 1087 * Reduces memory usage. 1088 * @param string $code 1089 */ 1090 public function unload( $code ) { 1091 unset( $this->data[$code] ); 1092 unset( $this->loadedItems[$code] ); 1093 unset( $this->loadedSubitems[$code] ); 1094 unset( $this->initialisedLangs[$code] ); 1095 unset( $this->shallowFallbacks[$code] ); 1096 1097 foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) { 1098 if ( $fbCode === $code ) { 1099 $this->unload( $shallowCode ); 1100 } 1101 } 1102 } 1103 1104 /** 1105 * Unload all data 1106 */ 1107 public function unloadAll() { 1108 foreach ( $this->initialisedLangs as $lang => $unused ) { 1109 $this->unload( $lang ); 1110 } 1111 } 1112 1113 /** 1114 * Disable the storage backend 1115 */ 1116 public function disableBackend() { 1117 $this->store = new LCStoreNull; 1118 $this->manualRecache = false; 1119 } 1120} 1121