1<?php /** @noinspection PhpStaticAsDynamicMethodCallInspection */ 2 3use Wikimedia\TestingAccessWrapper; 4 5/** 6 * @covers WANObjectCache::wrap 7 * @covers WANObjectCache::unwrap 8 * @covers WANObjectCache::worthRefreshExpiring 9 * @covers WANObjectCache::worthRefreshPopular 10 * @covers WANObjectCache::isValid 11 * @covers WANObjectCache::getWarmupKeyMisses 12 * @covers WANObjectCache::makeSisterKey 13 * @covers WANObjectCache::makeSisterKeys 14 * @covers WANObjectCache::getProcessCache 15 * @covers WANObjectCache::getNonProcessCachedMultiKeys 16 * @covers WANObjectCache::fetchWrappedValuesForWarmupCache 17 * @covers WANObjectCache::getInterimValue 18 * @covers WANObjectCache::setInterimValue 19 */ 20class WANObjectCacheTest extends PHPUnit\Framework\TestCase { 21 22 use MediaWikiCoversValidator; 23 24 /** 25 * @param array $params [optional] 26 * @return WANObjectCache[]|HashBagOStuff[] (WANObjectCache, BagOStuff) 27 */ 28 private function newWanCache( array $params = [] ) { 29 if ( isset( $params['broadcastRoutingPrefix'] ) ) { 30 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls 31 $bag = new McrouterHashBagOStuff(); 32 } else { 33 $bag = new HashBagOStuff(); 34 } 35 36 $cache = new WANObjectCache( [ 'cache' => $bag ] + $params ); 37 38 return [ $cache, $bag ]; 39 } 40 41 /** 42 * @dataProvider provideSetAndGet 43 * @covers WANObjectCache::set() 44 * @covers WANObjectCache::get() 45 * @covers WANObjectCache::makeKey() 46 */ 47 public function testSetAndGet( $value, $ttl ) { 48 list( $cache ) = $this->newWanCache(); 49 50 $curTTL = null; 51 $asOf = null; 52 $key = $cache->makeKey( 'x', wfRandomString() ); 53 54 $cache->get( $key, $curTTL, [], $asOf ); 55 $this->assertSame( null, $curTTL, "Current TTL (absent)" ); 56 $this->assertSame( null, $asOf, "Current as-of-time (absent)" ); 57 58 $t = microtime( true ); 59 60 $cache->set( $key, $value, $cache::TTL_UNCACHEABLE ); 61 $cache->get( $key, $curTTL, [], $asOf ); 62 $this->assertSame( null, $curTTL, "Current TTL (TTL_UNCACHEABLE)" ); 63 $this->assertSame( null, $asOf, "Current as-of-time (TTL_UNCACHEABLE)" ); 64 65 $cache->set( $key, $value, $ttl ); 66 67 $this->assertSame( $value, $cache->get( $key, $curTTL, [], $asOf ) ); 68 if ( $ttl === INF ) { 69 $this->assertSame( INF, $curTTL, "Current TTL" ); 70 } else { 71 $this->assertGreaterThan( 0, $curTTL, "Current TTL" ); 72 $this->assertLessThanOrEqual( $ttl, $curTTL, "Current TTL < nominal TTL" ); 73 } 74 $this->assertGreaterThanOrEqual( $t - 1, $asOf, "As-of-time in range of set() time" ); 75 $this->assertLessThanOrEqual( $t + 1, $asOf, "As-of-time in range of set() time" ); 76 } 77 78 public static function provideSetAndGet() { 79 return [ 80 // value, ttl 81 [ 14141, 3 ], 82 [ 3535.666, 3 ], 83 [ [], 3 ], 84 [ '0', 3 ], 85 [ (object)[ 'meow' ], 3 ], 86 [ INF, 3 ], 87 [ '', 3 ], 88 [ 'pizzacat', INF ], 89 [ null, 80 ] 90 ]; 91 } 92 93 /** 94 * @covers WANObjectCache::get() 95 * @covers WANObjectCache::makeGlobalKey() 96 */ 97 public function testGetNotExists() { 98 list( $cache ) = $this->newWanCache(); 99 100 $key = $cache->makeGlobalKey( 'y', wfRandomString(), 'p' ); 101 $curTTL = null; 102 $value = $cache->get( $key, $curTTL ); 103 104 $this->assertSame( false, $value, "Return value" ); 105 $this->assertSame( null, $curTTL, "current TTL" ); 106 } 107 108 /** 109 * @covers WANObjectCache::set() 110 */ 111 public function testSetOver() { 112 list( $cache ) = $this->newWanCache(); 113 114 $key = wfRandomString(); 115 for ( $i = 0; $i < 3; ++$i ) { 116 $value = wfRandomString(); 117 $cache->set( $key, $value, 3 ); 118 119 $this->assertSame( $cache->get( $key ), $value ); 120 } 121 } 122 123 public static function provideStaleSetParams() { 124 return [ 125 // Given a db transaction (trx lag) that started 30s ago, 126 // we generally don't want to cache its values. 127 [ 30, 0.0, false ], 128 [ 30, 2, false ], 129 [ 30, 10, false ], 130 [ 30, 20, false ], 131 // If the main reason we've hit 30s is that we spent 132 // a lot of time in the regeneration callback (as opposed 133 // to time mainly having passed before the cache computation) 134 // then cache it for at least a little while. 135 [ 30, 28, true ], 136 // Also if we don't know, cache it for a little while. 137 [ 30, null, true ], 138 ]; 139 } 140 141 /** 142 * @covers WANObjectCache::set() 143 * @dataProvider provideStaleSetParams 144 * @param int $ago 145 * @param float|null $walltime 146 * @param bool $cacheable 147 */ 148 public function testStaleSet( $ago, $walltime, $cacheable ) { 149 list( $cache ) = $this->newWanCache(); 150 $mockWallClock = 1549343530.2053; 151 $cache->setMockTime( $mockWallClock ); 152 153 $key = wfRandomString(); 154 $value = wfRandomString(); 155 156 $cache->set( 157 $key, 158 $value, 159 $cache::TTL_MINUTE, 160 [ 'since' => $mockWallClock - $ago, 'walltime' => $walltime ] 161 ); 162 163 $this->assertSame( 164 $cacheable ? $value : false, 165 $cache->get( $key ), 166 "Stale set() value ignored" 167 ); 168 } 169 170 /** 171 * @covers WANObjectCache::getWithSetCallback 172 */ 173 public function testProcessCacheTTL() { 174 list( $cache ) = $this->newWanCache(); 175 $mockWallClock = 1549343530.2053; 176 $cache->setMockTime( $mockWallClock ); 177 178 $key = "mykey-" . wfRandomString(); 179 180 $hits = 0; 181 $callback = static function ( $oldValue, &$ttl, &$setOpts ) use ( &$hits ) { 182 ++$hits; 183 return 42; 184 }; 185 186 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 187 $cache->delete( $key, $cache::HOLDOFF_TTL_NONE ); // clear persistent cache 188 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 189 $this->assertSame( 1, $hits, "Value process cached" ); 190 191 $mockWallClock += 6; 192 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 193 $this->assertSame( 2, $hits, "Value expired in process cache" ); 194 } 195 196 /** 197 * @covers WANObjectCache::getWithSetCallback 198 * @covers WANObjectCache::makeTombstonePurgeValue 199 */ 200 public function testProcessCacheLruAndDelete() { 201 list( $cache ) = $this->newWanCache(); 202 $mockWallClock = 1549343530.2053; 203 $cache->setMockTime( $mockWallClock ); 204 205 $hit = 0; 206 $fn = static function () use ( &$hit ) { 207 ++$hit; 208 return 42; 209 }; 210 $keysA = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 211 $keysB = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 212 $pcg = [ 'thiscache:1', 'thatcache:1', 'somecache:1' ]; 213 214 foreach ( $keysA as $i => $key ) { 215 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 216 } 217 $this->assertSame( 3, $hit, "Values not cached yet" ); 218 219 foreach ( $keysA as $i => $key ) { 220 // Should not evict from process cache 221 $cache->delete( $key ); 222 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 223 } 224 $this->assertSame( 3, $hit, "Values cached; not cleared by delete()" ); 225 226 foreach ( $keysB as $i => $key ) { 227 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 228 } 229 $this->assertSame( 6, $hit, "New values not cached yet" ); 230 231 foreach ( $keysB as $i => $key ) { 232 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 233 } 234 $this->assertSame( 6, $hit, "New values cached" ); 235 236 foreach ( $keysA as $i => $key ) { 237 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 238 } 239 $this->assertSame( 9, $hit, "Prior values evicted by new values" ); 240 } 241 242 /** 243 * @covers WANObjectCache::getWithSetCallback 244 */ 245 public function testProcessCacheInterimKeys() { 246 list( $cache ) = $this->newWanCache(); 247 $mockWallClock = 1549343530.2053; 248 $cache->setMockTime( $mockWallClock ); 249 250 $hit = 0; 251 $fn = static function () use ( &$hit ) { 252 ++$hit; 253 return 42; 254 }; 255 $keysA = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 256 $pcg = [ 'thiscache:1', 'thatcache:1', 'somecache:1' ]; 257 258 foreach ( $keysA as $i => $key ) { 259 $cache->delete( $key ); // tombstone key 260 $mockWallClock += 0.001; // cached values will be newer than tombstone 261 // Get into process cache (specific group) and interim cache 262 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 263 } 264 $this->assertSame( 3, $hit ); 265 266 // Get into process cache (default group) 267 $key = reset( $keysA ); 268 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 269 $this->assertSame( 3, $hit, "Value recently interim-cached" ); 270 271 $mockWallClock += 0.2; // interim key not brand new 272 $cache->clearProcessCache(); 273 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 274 $this->assertSame( 4, $hit, "Value calculated (interim key not recent and reset)" ); 275 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 276 $this->assertSame( 4, $hit, "Value process cached" ); 277 } 278 279 /** 280 * @covers WANObjectCache::getWithSetCallback 281 */ 282 public function testProcessCacheNesting() { 283 list( $cache ) = $this->newWanCache(); 284 $mockWallClock = 1549343530.2053; 285 $cache->setMockTime( $mockWallClock ); 286 287 $keyOuter = "outer-" . wfRandomString(); 288 $keyInner = "inner-" . wfRandomString(); 289 290 $innerHit = 0; 291 $innerFn = static function () use ( &$innerHit ) { 292 ++$innerHit; 293 return 42; 294 }; 295 296 $outerHit = 0; 297 $outerFn = static function () use ( $keyInner, $innerFn, $cache, &$outerHit ) { 298 ++$outerHit; 299 $v = $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 300 301 return 43 + $v; 302 }; 303 304 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 305 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 306 307 $this->assertSame( 1, $innerHit, "Inner callback value cached" ); 308 $cache->delete( $keyInner, $cache::HOLDOFF_TTL_NONE ); 309 $mockWallClock += 1; 310 311 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 312 $this->assertSame( 1, $innerHit, "Inner callback process cached" ); 313 314 // Outer key misses and inner key process cache value is refused 315 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 316 317 $this->assertSame( 1, $outerHit, "Outer callback value not yet cached" ); 318 $this->assertSame( 2, $innerHit, "Inner callback value process cache skipped" ); 319 320 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 321 322 $this->assertSame( 1, $outerHit, "Outer callback value cached" ); 323 324 $cache->delete( $keyInner, $cache::HOLDOFF_TTL_NONE ); 325 $cache->delete( $keyOuter, $cache::HOLDOFF_TTL_NONE ); 326 $mockWallClock += 1; 327 $cache->clearProcessCache(); 328 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 329 330 $this->assertSame( 2, $outerHit, "Outer callback value not yet cached" ); 331 $this->assertSame( 3, $innerHit, "Inner callback value not yet cached" ); 332 333 $cache->delete( $keyInner, $cache::HOLDOFF_TTL_NONE ); 334 $mockWallClock += 1; 335 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 336 337 $this->assertSame( 3, $innerHit, "Inner callback value process cached" ); 338 } 339 340 /** 341 * @dataProvider getWithSetCallback_provider 342 * @covers WANObjectCache::getWithSetCallback() 343 * @covers WANObjectCache::fetchOrRegenerate() 344 * @param array $extOpts 345 */ 346 public function testGetWithSetCallback( array $extOpts ) { 347 list( $cache ) = $this->newWanCache(); 348 349 $key = wfRandomString(); 350 $value = wfRandomString(); 351 $cKey1 = wfRandomString(); 352 $cKey2 = wfRandomString(); 353 354 $priorValue = null; 355 $priorAsOf = null; 356 $wasSet = 0; 357 $func = static function ( $old, &$ttl, &$opts, $asOf ) 358 use ( &$wasSet, &$priorValue, &$priorAsOf, $value ) { 359 ++$wasSet; 360 $priorValue = $old; 361 $priorAsOf = $asOf; 362 $ttl = 20; // override with another value 363 return $value; 364 }; 365 366 $mockWallClock = 1549343530.2053; 367 $priorTime = $mockWallClock; // reference time 368 $cache->setMockTime( $mockWallClock ); 369 370 $wasSet = 0; 371 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] + $extOpts ); 372 $this->assertSame( $value, $v, "Value returned" ); 373 $this->assertSame( 1, $wasSet, "Value regenerated" ); 374 $this->assertSame( false, $priorValue, "No prior value" ); 375 $this->assertSame( null, $priorAsOf, "No prior value" ); 376 377 $curTTL = null; 378 $cache->get( $key, $curTTL ); 379 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 380 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 381 382 $wasSet = 0; 383 $v = $cache->getWithSetCallback( 384 $key, 30, $func, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 385 $this->assertSame( $value, $v, "Value returned" ); 386 $this->assertSame( 0, $wasSet, "Value not regenerated" ); 387 388 $mockWallClock += 1; 389 390 $wasSet = 0; 391 $v = $cache->getWithSetCallback( 392 $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 393 ); 394 $this->assertSame( $value, $v, "Value returned" ); 395 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 396 $this->assertSame( $value, $priorValue, "Has prior value" ); 397 $this->assertIsFloat( $priorAsOf, "Has prior value" ); 398 $t1 = $cache->getCheckKeyTime( $cKey1 ); 399 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 400 $t2 = $cache->getCheckKeyTime( $cKey2 ); 401 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 402 403 $mockWallClock += 0.2; // interim key is not brand new and check keys have past values 404 $priorTime = $mockWallClock; // reference time 405 $wasSet = 0; 406 $v = $cache->getWithSetCallback( 407 $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 408 ); 409 $this->assertSame( $value, $v, "Value returned" ); 410 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 411 $t1 = $cache->getCheckKeyTime( $cKey1 ); 412 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 413 $t2 = $cache->getCheckKeyTime( $cKey2 ); 414 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 415 416 $curTTL = null; 417 $v = $cache->get( $key, $curTTL, [ $cKey1, $cKey2 ] ); 418 $this->assertSame( $value, $v, "Value returned" ); 419 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 420 421 $wasSet = 0; 422 $key = wfRandomString(); 423 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts ); 424 $this->assertSame( $value, $v, "Value returned" ); 425 $cache->delete( $key ); 426 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts ); 427 $this->assertSame( $value, $v, "Value still returned after deleted" ); 428 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 429 430 $oldValReceived = -1; 431 $oldAsOfReceived = -1; 432 $checkFunc = static function ( $oldVal, &$ttl, array $setOpts, $oldAsOf ) 433 use ( &$oldValReceived, &$oldAsOfReceived, &$wasSet ) { 434 ++$wasSet; 435 $oldValReceived = $oldVal; 436 $oldAsOfReceived = $oldAsOf; 437 438 return 'xxx' . $wasSet; 439 }; 440 441 $mockWallClock = 1549343530.2053; 442 $priorTime = $mockWallClock; // reference time 443 444 $wasSet = 0; 445 $key = wfRandomString(); 446 $v = $cache->getWithSetCallback( 447 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 448 $this->assertSame( 'xxx1', $v, "Value returned" ); 449 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 450 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 451 452 $mockWallClock += 40; 453 $v = $cache->getWithSetCallback( 454 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 455 $this->assertSame( 'xxx2', $v, "Value still returned after expired" ); 456 $this->assertSame( 2, $wasSet, "Value recalculated while expired" ); 457 $this->assertSame( 'xxx1', $oldValReceived, "Callback got stale value" ); 458 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got stale value" ); 459 460 $mockWallClock += 260; 461 $v = $cache->getWithSetCallback( 462 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 463 $this->assertSame( 'xxx3', $v, "Value still returned after expired" ); 464 $this->assertSame( 3, $wasSet, "Value recalculated while expired" ); 465 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 466 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 467 468 $mockWallClock = ( $priorTime - $cache::HOLDOFF_TTL - 1 ); 469 $wasSet = 0; 470 $key = wfRandomString(); 471 $checkKey = $cache->makeKey( 'template', 'X' ); 472 $cache->touchCheckKey( $checkKey ); // init check key 473 $mockWallClock = $priorTime; 474 $v = $cache->getWithSetCallback( 475 $key, 476 $cache::TTL_INDEFINITE, 477 $checkFunc, 478 [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts 479 ); 480 $this->assertSame( 'xxx1', $v, "Value returned" ); 481 $this->assertSame( 1, $wasSet, "Value computed" ); 482 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 483 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 484 485 $mockWallClock += $cache::TTL_HOUR; // some time passes 486 $v = $cache->getWithSetCallback( 487 $key, 488 $cache::TTL_INDEFINITE, 489 $checkFunc, 490 [ 491 'graceTTL' => $cache::TTL_WEEK, 492 'checkKeys' => [ $checkKey ], 493 'ageNew' => -1 494 ] + $extOpts 495 ); 496 $this->assertSame( 'xxx1', $v, "Cached value returned" ); 497 $this->assertSame( 1, $wasSet, "Cached value returned" ); 498 499 $cache->touchCheckKey( $checkKey ); // make key stale 500 $mockWallClock += 0.01; // ~1 week left of grace (barely stale to avoid refreshes) 501 502 $v = $cache->getWithSetCallback( 503 $key, 504 $cache::TTL_INDEFINITE, 505 $checkFunc, 506 [ 507 'graceTTL' => $cache::TTL_WEEK, 508 'checkKeys' => [ $checkKey ], 509 'ageNew' => -1, 510 ] + $extOpts 511 ); 512 $this->assertSame( 'xxx1', $v, "Value still returned after expired (in grace)" ); 513 $this->assertSame( 1, $wasSet, "Value still returned after expired (in grace)" ); 514 515 // Chance of refresh increase to unity as staleness approaches graceTTL 516 $mockWallClock += $cache::TTL_WEEK; // 8 days of being stale 517 $v = $cache->getWithSetCallback( 518 $key, 519 $cache::TTL_INDEFINITE, 520 $checkFunc, 521 [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts 522 ); 523 $this->assertSame( 'xxx2', $v, "Value was recomputed (past grace)" ); 524 $this->assertSame( 2, $wasSet, "Value was recomputed (past grace)" ); 525 $this->assertSame( 'xxx1', $oldValReceived, "Callback got post-grace stale value" ); 526 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got post-grace stale value" ); 527 } 528 529 /** 530 * @dataProvider getWithSetCallback_provider 531 * @covers WANObjectCache::getWithSetCallback() 532 * @covers WANObjectCache::fetchOrRegenerate() 533 * @param array $extOpts 534 */ 535 public function testGetWithSetCallback_touched( array $extOpts ) { 536 list( $cache ) = $this->newWanCache(); 537 538 $mockWallClock = 1549343530.2053; 539 $cache->setMockTime( $mockWallClock ); 540 541 $checkFunc = static function ( $oldVal, &$ttl, array $setOpts, $oldAsOf ) 542 use ( &$wasSet ) { 543 ++$wasSet; 544 545 return 'xxx' . $wasSet; 546 }; 547 548 $key = wfRandomString(); 549 $wasSet = 0; 550 $touched = null; 551 $touchedCallback = static function () use ( &$touched ) { 552 return $touched; 553 }; 554 $v = $cache->getWithSetCallback( 555 $key, 556 $cache::TTL_INDEFINITE, 557 $checkFunc, 558 [ 'touchedCallback' => $touchedCallback ] + $extOpts 559 ); 560 $mockWallClock += 60; 561 $v = $cache->getWithSetCallback( 562 $key, 563 $cache::TTL_INDEFINITE, 564 $checkFunc, 565 [ 'touchedCallback' => $touchedCallback ] + $extOpts 566 ); 567 $this->assertSame( 'xxx1', $v, "Value was computed once" ); 568 $this->assertSame( 1, $wasSet, "Value was computed once" ); 569 570 $touched = $mockWallClock - 10; 571 $v = $cache->getWithSetCallback( 572 $key, 573 $cache::TTL_INDEFINITE, 574 $checkFunc, 575 [ 'touchedCallback' => $touchedCallback ] + $extOpts 576 ); 577 $v = $cache->getWithSetCallback( 578 $key, 579 $cache::TTL_INDEFINITE, 580 $checkFunc, 581 [ 'touchedCallback' => $touchedCallback ] + $extOpts 582 ); 583 $this->assertSame( 'xxx2', $v, "Value was recomputed once" ); 584 $this->assertSame( 2, $wasSet, "Value was recomputed once" ); 585 } 586 587 public static function getWithSetCallback_provider() { 588 return [ 589 [ [], false ], 590 [ [ 'version' => 1 ], true ] 591 ]; 592 } 593 594 public function testPreemtiveRefresh() { 595 $value = 'KatCafe'; 596 $wasSet = 0; 597 $func = static function ( $old, &$ttl, &$opts, $asOf ) use ( &$wasSet, &$value ) 598 { 599 ++$wasSet; 600 return $value; 601 }; 602 603 $cache = new NearExpiringWANObjectCache( [ 'cache' => new HashBagOStuff() ] ); 604 $mockWallClock = 1549343530.2053; 605 $cache->setMockTime( $mockWallClock ); 606 607 $wasSet = 0; 608 $key = wfRandomString(); 609 $opts = [ 'lowTTL' => 30 ]; 610 $v = $cache->getWithSetCallback( $key, 20, $func, $opts ); 611 $this->assertSame( $value, $v, "Value returned" ); 612 $this->assertSame( 1, $wasSet, "Value calculated" ); 613 614 $mockWallClock += 0.2; // interim key is not brand new 615 $v = $cache->getWithSetCallback( $key, 20, $func, $opts ); 616 $this->assertSame( 2, $wasSet, "Value re-calculated" ); 617 618 $wasSet = 0; 619 $key = wfRandomString(); 620 $opts = [ 'lowTTL' => 1 ]; 621 $v = $cache->getWithSetCallback( $key, 30, $func, $opts ); 622 $this->assertSame( $value, $v, "Value returned" ); 623 $this->assertSame( 1, $wasSet, "Value calculated" ); 624 $v = $cache->getWithSetCallback( $key, 30, $func, $opts ); 625 $this->assertSame( 1, $wasSet, "Value cached" ); 626 627 $asycList = []; 628 $asyncHandler = static function ( $callback ) use ( &$asycList ) { 629 $asycList[] = $callback; 630 }; 631 $cache = new NearExpiringWANObjectCache( [ 632 'cache' => new HashBagOStuff(), 633 'asyncHandler' => $asyncHandler 634 ] ); 635 636 $mockWallClock = 1549343530.2053; 637 $priorTime = $mockWallClock; // reference time 638 $cache->setMockTime( $mockWallClock ); 639 640 $wasSet = 0; 641 $key = wfRandomString(); 642 $opts = [ 'lowTTL' => 100 ]; 643 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 644 $this->assertSame( $value, $v, "Value returned" ); 645 $this->assertSame( 1, $wasSet, "Value calculated" ); 646 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 647 $this->assertSame( 1, $wasSet, "Cached value used" ); 648 $this->assertSame( $v, $value, "Value cached" ); 649 650 $mockWallClock += 250; 651 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 652 $this->assertSame( $value, $v, "Value returned" ); 653 $this->assertSame( 1, $wasSet, "Stale value used" ); 654 $this->assertCount( 1, $asycList, "Refresh deferred." ); 655 $value = 'NewCatsInTown'; // change callback return value 656 $asycList[0](); // run the refresh callback 657 $asycList = []; 658 $this->assertSame( 2, $wasSet, "Value calculated at later time" ); 659 $this->assertSame( [], $asycList, "No deferred refreshes added." ); 660 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 661 $this->assertSame( $value, $v, "New value stored" ); 662 663 $cache = new PopularityRefreshingWANObjectCache( [ 664 'cache' => new HashBagOStuff() 665 ] ); 666 667 $mockWallClock = $priorTime; 668 $cache->setMockTime( $mockWallClock ); 669 670 $wasSet = 0; 671 $key = wfRandomString(); 672 $opts = [ 'hotTTR' => 900 ]; 673 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 674 $this->assertSame( $value, $v, "Value returned" ); 675 $this->assertSame( 1, $wasSet, "Value calculated" ); 676 677 $mockWallClock += 30; 678 679 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 680 $this->assertSame( 1, $wasSet, "Value cached" ); 681 682 $mockWallClock = $priorTime; 683 $wasSet = 0; 684 $key = wfRandomString(); 685 $opts = [ 'hotTTR' => 10 ]; 686 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 687 $this->assertSame( $value, $v, "Value returned" ); 688 $this->assertSame( 1, $wasSet, "Value calculated" ); 689 690 $mockWallClock += 30; 691 692 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 693 $this->assertSame( $value, $v, "Value returned" ); 694 $this->assertSame( 2, $wasSet, "Value re-calculated" ); 695 } 696 697 /** 698 * @dataProvider getMultiWithSetCallback_provider 699 * @covers WANObjectCache::getMultiWithSetCallback 700 * @covers WANObjectCache::makeMultiKeys 701 * @covers WANObjectCache::getMulti 702 * @param array $extOpts 703 */ 704 public function testGetMultiWithSetCallback( array $extOpts ) { 705 list( $cache ) = $this->newWanCache(); 706 707 $keyA = wfRandomString(); 708 $keyB = wfRandomString(); 709 $keyC = wfRandomString(); 710 $cKey1 = wfRandomString(); 711 $cKey2 = wfRandomString(); 712 713 $priorValue = null; 714 $priorAsOf = null; 715 $wasSet = 0; 716 $genFunc = static function ( $id, $old, &$ttl, &$opts, $asOf ) use ( 717 &$wasSet, &$priorValue, &$priorAsOf 718 ) { 719 ++$wasSet; 720 $priorValue = $old; 721 $priorAsOf = $asOf; 722 $ttl = 20; // override with another value 723 return "@$id$"; 724 }; 725 726 $mockWallClock = 1549343530.2053; 727 $priorTime = $mockWallClock; // reference time 728 $cache->setMockTime( $mockWallClock ); 729 730 $wasSet = 0; 731 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); 732 $value = "@3353$"; 733 $v = $cache->getMultiWithSetCallback( 734 $keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts ); 735 $this->assertSame( $value, $v[$keyA], "Value returned" ); 736 $this->assertSame( 1, $wasSet, "Value regenerated" ); 737 $this->assertSame( false, $priorValue, "No prior value" ); 738 $this->assertSame( null, $priorAsOf, "No prior value" ); 739 740 $curTTL = null; 741 $cache->get( $keyA, $curTTL ); 742 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 743 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 744 745 $wasSet = 0; 746 $value = "@efef$"; 747 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 748 $v = $cache->getMultiWithSetCallback( 749 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 750 $this->assertSame( $value, $v[$keyB], "Value returned" ); 751 $this->assertSame( 1, $wasSet, "Value regenerated" ); 752 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 753 754 $v = $cache->getMultiWithSetCallback( 755 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 756 $this->assertSame( $value, $v[$keyB], "Value returned" ); 757 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 758 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 759 760 $mockWallClock += 1; 761 762 $wasSet = 0; 763 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 764 $v = $cache->getMultiWithSetCallback( 765 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 766 ); 767 $this->assertSame( $value, $v[$keyB], "Value returned" ); 768 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 769 $this->assertSame( $value, $priorValue, "Has prior value" ); 770 $this->assertIsFloat( $priorAsOf, "Has prior value" ); 771 $t1 = $cache->getCheckKeyTime( $cKey1 ); 772 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 773 $t2 = $cache->getCheckKeyTime( $cKey2 ); 774 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 775 776 $mockWallClock += 0.01; 777 $priorTime = $mockWallClock; 778 $value = "@43636$"; 779 $wasSet = 0; 780 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); 781 $v = $cache->getMultiWithSetCallback( 782 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 783 ); 784 $this->assertSame( $value, $v[$keyC], "Value returned" ); 785 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 786 $t1 = $cache->getCheckKeyTime( $cKey1 ); 787 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 788 $t2 = $cache->getCheckKeyTime( $cKey2 ); 789 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 790 791 $curTTL = null; 792 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); 793 $this->assertSame( $value, $v, "Value returned" ); 794 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 795 796 $wasSet = 0; 797 $key = wfRandomString(); 798 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 799 $v = $cache->getMultiWithSetCallback( 800 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 801 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); 802 $cache->delete( $key ); 803 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 804 $v = $cache->getMultiWithSetCallback( 805 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 806 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); 807 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 808 809 $calls = 0; 810 $ids = [ 1, 2, 3, 4, 5, 6 ]; 811 $keyFunc = static function ( $id, WANObjectCache $wanCache ) { 812 return $wanCache->makeKey( 'test', $id ); 813 }; 814 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); 815 $genFunc = static function ( $id, $oldValue, &$ttl, array &$setops ) use ( &$calls ) { 816 ++$calls; 817 818 return "val-{$id}"; 819 }; 820 $values = $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); 821 822 $this->assertSame( 823 [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], 824 array_values( $values ), 825 "Correct values in correct order" 826 ); 827 $this->assertSame( 828 array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ), 829 array_keys( $values ), 830 "Correct keys in correct order" 831 ); 832 $this->assertSame( count( $ids ), $calls ); 833 834 $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); 835 $this->assertSame( count( $ids ), $calls, "Values cached" ); 836 837 // Mock the BagOStuff to assure only one getMulti() call given process caching 838 $localBag = $this->getMockBuilder( HashBagOStuff::class ) 839 ->setMethods( [ 'getMulti' ] )->getMock(); 840 $localBag->expects( $this->exactly( 1 ) )->method( 'getMulti' )->willReturn( [ 841 'WANCache:v:' . 'k1' => 'val-id1', 842 'WANCache:v:' . 'k2' => 'val-id2' 843 ] ); 844 $wanCache = new WANObjectCache( [ 'cache' => $localBag ] ); 845 846 // Warm the process cache 847 $keyedIds = new ArrayIterator( [ 'k1' => 'id1', 'k2' => 'id2' ] ); 848 $this->assertSame( 849 [ 'k1' => 'val-id1', 'k2' => 'val-id2' ], 850 $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] ) 851 ); 852 // Use the process cache 853 $this->assertSame( 854 [ 'k1' => 'val-id1', 'k2' => 'val-id2' ], 855 $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] ) 856 ); 857 } 858 859 public static function getMultiWithSetCallback_provider() { 860 return [ 861 [ [], false ], 862 [ [ 'version' => 1 ], true ] 863 ]; 864 } 865 866 /** 867 * @dataProvider getMultiWithSetCallbackRefresh_provider 868 * @param bool $expiring 869 * @param bool $popular 870 * @param array $idsByKey 871 */ 872 public function testGetMultiWithSetCallbackRefresh( $expiring, $popular, array $idsByKey ) { 873 $deferredCbs = []; 874 $bag = new HashBagOStuff(); 875 $cache = $this->getMockBuilder( WANObjectCache::class ) 876 ->setMethods( [ 'worthRefreshExpiring', 'worthRefreshPopular' ] ) 877 ->setConstructorArgs( [ 878 [ 879 'cache' => $bag, 880 'asyncHandler' => static function ( $callback ) use ( &$deferredCbs ) { 881 $deferredCbs[] = $callback; 882 } 883 ] 884 ] ) 885 ->getMock(); 886 887 $cache->method( 'worthRefreshExpiring' )->willReturn( $expiring ); 888 $cache->method( 'worthRefreshPopular' )->willReturn( $popular ); 889 890 $wasSet = 0; 891 $keyedIds = new ArrayIterator( $idsByKey ); 892 $genFunc = static function ( $id, $old, &$ttl, &$opts, $asOf ) use ( &$wasSet ) { 893 ++$wasSet; 894 $ttl = 20; // override with another value 895 return "@$id$"; 896 }; 897 898 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 899 $this->assertSame( count( $idsByKey ), $wasSet, "Initial sets" ); 900 $this->assertSame( [], $deferredCbs, "No deferred callbacks yet" ); 901 foreach ( $idsByKey as $key => $id ) { 902 $this->assertSame( "@$id$", $v[$key], "Initial cache value generation" ); 903 } 904 905 $wasSet = 0; 906 $preemptiveRefresh = ( $expiring || $popular ); 907 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 908 $this->assertSame( 0, $wasSet, "No values generated" ); 909 $this->assertCount( 910 $preemptiveRefresh ? count( $idsByKey ) : 0, 911 $deferredCbs, 912 "Deferred callbacks queued" 913 ); 914 foreach ( $idsByKey as $key => $id ) { 915 $this->assertSame( "@$id$", $v[$key], "Cached value reused; refresh scheduled" ); 916 } 917 918 // Run the deferred callbacks... 919 $deferredCbsReady = $deferredCbs; 920 $deferredCbs = []; // empty by-reference queue 921 foreach ( $deferredCbsReady as $deferredCb ) { 922 $deferredCb(); 923 } 924 925 $this->assertSame( 926 ( $preemptiveRefresh ? count( $idsByKey ) : 0 ), 927 $wasSet, 928 "Deferred callback regenerations" 929 ); 930 $this->assertSame( [], $deferredCbs, "Deferred callbacks queue empty" ); 931 932 $wasSet = 0; 933 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 934 $this->assertSame( 935 0, 936 $wasSet, 937 "Deferred callbacks did not run again" 938 ); 939 foreach ( $idsByKey as $key => $id ) { 940 $this->assertSame( "@$id$", $v[$key], "Cached value OK after deferred refresh run" ); 941 } 942 } 943 944 public static function getMultiWithSetCallbackRefresh_provider() { 945 return [ 946 [ true, true, [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ] ], 947 [ true, false, [ 'a' => 'x', 'b' => 'y', 'c' => 'z', 'd' => 'w' ] ], 948 [ false, true, [ 'a' => 'p', 'b' => 'q', 'c' => 'r', 'd' => 's' ] ], 949 [ false, false, [ 'a' => '%', 'b' => '^', 'c' => '&', 'd' => 'ç' ] ] 950 ]; 951 } 952 953 /** 954 * @dataProvider getMultiWithUnionSetCallback_provider 955 * @covers WANObjectCache::getMultiWithUnionSetCallback() 956 * @covers WANObjectCache::makeMultiKeys() 957 * @param array $extOpts 958 */ 959 public function testGetMultiWithUnionSetCallback( array $extOpts ) { 960 list( $cache ) = $this->newWanCache(); 961 962 $keyA = wfRandomString(); 963 $keyB = wfRandomString(); 964 $keyC = wfRandomString(); 965 $cKey1 = wfRandomString(); 966 $cKey2 = wfRandomString(); 967 968 $wasSet = 0; 969 $genFunc = static function ( array $ids, array &$ttls, array &$setOpts ) use ( 970 &$wasSet, &$priorValue, &$priorAsOf 971 ) { 972 $newValues = []; 973 foreach ( $ids as $id ) { 974 ++$wasSet; 975 $newValues[$id] = "@$id$"; 976 $ttls[$id] = 20; // override with another value 977 } 978 979 return $newValues; 980 }; 981 982 $mockWallClock = 1549343530.2053; 983 $priorTime = $mockWallClock; // reference time 984 $cache->setMockTime( $mockWallClock ); 985 986 $wasSet = 0; 987 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); 988 $value = "@3353$"; 989 $v = $cache->getMultiWithUnionSetCallback( 990 $keyedIds, 30, $genFunc, $extOpts ); 991 $this->assertSame( $value, $v[$keyA], "Value returned" ); 992 $this->assertSame( 1, $wasSet, "Value regenerated" ); 993 994 $curTTL = null; 995 $cache->get( $keyA, $curTTL ); 996 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 997 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 998 999 $wasSet = 0; 1000 $value = "@efef$"; 1001 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 1002 $v = $cache->getMultiWithUnionSetCallback( 1003 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); 1004 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1005 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1006 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 1007 1008 $v = $cache->getMultiWithUnionSetCallback( 1009 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); 1010 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1011 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 1012 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 1013 1014 $mockWallClock += 1; 1015 1016 $wasSet = 0; 1017 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 1018 $v = $cache->getMultiWithUnionSetCallback( 1019 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 1020 ); 1021 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1022 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 1023 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1024 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 1025 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1026 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 1027 1028 $mockWallClock += 0.01; 1029 $priorTime = $mockWallClock; 1030 $value = "@43636$"; 1031 $wasSet = 0; 1032 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); 1033 $v = $cache->getMultiWithUnionSetCallback( 1034 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 1035 ); 1036 $this->assertSame( $value, $v[$keyC], "Value returned" ); 1037 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 1038 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1039 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 1040 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1041 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 1042 1043 $curTTL = null; 1044 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); 1045 $this->assertSame( $value, $v, "Value returned" ); 1046 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 1047 1048 $wasSet = 0; 1049 $key = wfRandomString(); 1050 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 1051 $v = $cache->getMultiWithUnionSetCallback( 1052 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 1053 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); 1054 $cache->delete( $key ); 1055 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 1056 $v = $cache->getMultiWithUnionSetCallback( 1057 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 1058 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); 1059 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 1060 1061 $calls = 0; 1062 $ids = [ 1, 2, 3, 4, 5, 6 ]; 1063 $keyFunc = static function ( $id, WANObjectCache $wanCache ) { 1064 return $wanCache->makeKey( 'test', $id ); 1065 }; 1066 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); 1067 $genFunc = static function ( array $ids, array &$ttls, array &$setOpts ) use ( &$calls ) { 1068 $newValues = []; 1069 foreach ( $ids as $id ) { 1070 ++$calls; 1071 $newValues[$id] = "val-{$id}"; 1072 } 1073 1074 return $newValues; 1075 }; 1076 $values = $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); 1077 1078 $this->assertSame( 1079 [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], 1080 array_values( $values ), 1081 "Correct values in correct order" 1082 ); 1083 $this->assertSame( 1084 array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ), 1085 array_keys( $values ), 1086 "Correct keys in correct order" 1087 ); 1088 $this->assertSame( count( $ids ), $calls ); 1089 1090 $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); 1091 $this->assertSame( count( $ids ), $calls, "Values cached" ); 1092 } 1093 1094 public static function getMultiWithUnionSetCallback_provider() { 1095 return [ 1096 [ [], false ], 1097 [ [ 'version' => 1 ], true ] 1098 ]; 1099 } 1100 1101 public static function provideCoalesceAndMcrouterSettings() { 1102 return [ 1103 [ [ 'coalesceScheme' => 'hash_tag' ], '{' ], 1104 [ [ 'broadcastRoutingPrefix' => '/*/test/', 'coalesceScheme' => 'hash_stop' ], '|#|' ], 1105 ]; 1106 } 1107 1108 /** 1109 * @dataProvider getMultiWithUnionSetCallbackRefresh_provider 1110 * @param bool $expiring 1111 * @param bool $popular 1112 * @param array $idsByKey 1113 */ 1114 public function testGetMultiWithUnionSetCallbackRefresh( $expiring, $popular, array $idsByKey ) { 1115 $deferredCbs = []; 1116 $bag = new HashBagOStuff(); 1117 $cache = $this->getMockBuilder( WANObjectCache::class ) 1118 ->setMethods( [ 'worthRefreshExpiring', 'worthRefreshPopular' ] ) 1119 ->setConstructorArgs( [ 1120 [ 1121 'cache' => $bag, 1122 'asyncHandler' => static function ( $callback ) use ( &$deferredCbs ) { 1123 $deferredCbs[] = $callback; 1124 } 1125 ] 1126 ] ) 1127 ->getMock(); 1128 1129 $cache->expects( $this->any() )->method( 'worthRefreshExpiring' )->willReturn( $expiring ); 1130 $cache->expects( $this->any() )->method( 'worthRefreshPopular' )->willReturn( $popular ); 1131 1132 $wasSet = 0; 1133 $keyedIds = new ArrayIterator( $idsByKey ); 1134 $genFunc = static function ( array $ids, array &$ttls, array &$setOpts ) use ( &$wasSet ) { 1135 $newValues = []; 1136 foreach ( $ids as $id ) { 1137 ++$wasSet; 1138 $newValues[$id] = "@$id$"; 1139 $ttls[$id] = 20; // override with another value 1140 } 1141 1142 return $newValues; 1143 }; 1144 1145 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1146 $this->assertSame( count( $idsByKey ), $wasSet, "Initial sets" ); 1147 $this->assertSame( [], $deferredCbs, "No deferred callbacks yet" ); 1148 foreach ( $idsByKey as $key => $id ) { 1149 $this->assertSame( "@$id$", $v[$key], "Initial cache value generation" ); 1150 } 1151 1152 $preemptiveRefresh = ( $expiring || $popular ); 1153 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1154 $this->assertSame( count( $idsByKey ), $wasSet, "Deferred callbacks did not run yet" ); 1155 $this->assertCount( 1156 $preemptiveRefresh ? count( $idsByKey ) : 0, 1157 $deferredCbs, 1158 "Deferred callbacks queued" 1159 ); 1160 foreach ( $idsByKey as $key => $id ) { 1161 $this->assertSame( "@$id$", $v[$key], "Cached value reused; refresh scheduled" ); 1162 } 1163 1164 // Run the deferred callbacks... 1165 $deferredCbsReady = $deferredCbs; 1166 $deferredCbs = []; // empty by-reference queue 1167 foreach ( $deferredCbsReady as $deferredCb ) { 1168 $deferredCb(); 1169 } 1170 1171 $this->assertSame( 1172 count( $idsByKey ) * ( $preemptiveRefresh ? 2 : 1 ), 1173 $wasSet, 1174 "Deferred callback regenerations" 1175 ); 1176 $this->assertSame( [], $deferredCbs, "Deferred callbacks queue empty" ); 1177 1178 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1179 $this->assertSame( 1180 count( $idsByKey ) * ( $preemptiveRefresh ? 2 : 1 ), 1181 $wasSet, 1182 "Deferred callbacks did not run again yet" 1183 ); 1184 foreach ( $idsByKey as $key => $id ) { 1185 $this->assertSame( "@$id$", $v[$key], "Cached value OK after deferred refresh run" ); 1186 } 1187 } 1188 1189 public static function getMultiWithUnionSetCallbackRefresh_provider() { 1190 return [ 1191 [ true, true, [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ] ], 1192 [ true, false, [ 'a' => 'x', 'b' => 'y', 'c' => 'z', 'd' => 'w' ] ], 1193 [ false, true, [ 'a' => 'p', 'b' => 'q', 'c' => 'r', 'd' => 's' ] ], 1194 [ false, false, [ 'a' => '%', 'b' => '^', 'c' => '&', 'd' => 'ç' ] ] 1195 ]; 1196 } 1197 1198 /** 1199 * @covers WANObjectCache::getWithSetCallback() 1200 * @covers WANObjectCache::fetchOrRegenerate() 1201 * @dataProvider provideCoalesceAndMcrouterSettings 1202 */ 1203 public function testLockTSE( array $params ) { 1204 list( $cache, $bag ) = $this->newWanCache( $params ); 1205 $key = wfRandomString(); 1206 $value = wfRandomString(); 1207 1208 $mockWallClock = 1549343530.2053; 1209 $cache->setMockTime( $mockWallClock ); 1210 1211 $calls = 0; 1212 $func = static function () use ( &$calls, $value, $cache, $key ) { 1213 ++$calls; 1214 return $value; 1215 }; 1216 1217 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] ); 1218 $this->assertSame( $value, $ret ); 1219 $this->assertSame( 1, $calls, 'Value was populated' ); 1220 1221 // Acquire the mutex to verify that getWithSetCallback uses lockTSE properly 1222 $this->setMutexKey( $bag, $key ); 1223 1224 $checkKeys = [ wfRandomString() ]; // new check keys => force misses 1225 $ret = $cache->getWithSetCallback( $key, 30, $func, 1226 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1227 $this->assertSame( $value, $ret, 'Old value used' ); 1228 $this->assertSame( 1, $calls, 'Callback was not used' ); 1229 1230 $cache->delete( $key ); // no value at all anymore and still locked 1231 1232 $mockWallClock += 0.001; // cached values will be newer than tombstone 1233 $ret = $cache->getWithSetCallback( $key, 30, $func, 1234 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1235 $this->assertSame( $value, $ret, 'Callback was used; interim saved' ); 1236 $this->assertSame( 2, $calls, 'Callback was used; interim saved' ); 1237 1238 $ret = $cache->getWithSetCallback( $key, 30, $func, 1239 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1240 $this->assertSame( $value, $ret, 'Callback was not used; used interim (mutex failed)' ); 1241 $this->assertSame( 2, $calls, 'Callback was not used; used interim (mutex failed)' ); 1242 } 1243 1244 private function setMutexKey( BagOStuff $bag, $key ) { 1245 // Cover all formats for "coalesceScheme" 1246 $bag->add( "WANCache:$key|#|m", 1 ); 1247 $bag->add( "WANCache:{" . $key . "}:m", 1 ); 1248 } 1249 1250 private function clearMutexKey( BagOStuff $bag, $key ) { 1251 // Cover all formats for "coalesceScheme" 1252 $bag->delete( "WANCache:$key|#|m" ); 1253 $bag->delete( "WANCache:{" . $key . "}:m" ); 1254 } 1255 1256 private function setCheckKey( BagOStuff $bag, $key, $time ) { 1257 // Cover all formats for "coalesceScheme" 1258 $bag->set( "WANCache:$key|#|t", "PURGED:$time" ); 1259 $bag->set( "WANCache:{" . $key . "}:t", "PURGED:$time" ); 1260 } 1261 1262 /** 1263 * @covers WANObjectCache::getWithSetCallback() 1264 * @covers WANObjectCache::fetchOrRegenerate() 1265 * @covers WANObjectCache::set() 1266 * @dataProvider provideCoalesceAndMcrouterSettings 1267 */ 1268 public function testLockTSESlow( array $params ) { 1269 list( $cache, $bag ) = $this->newWanCache( $params ); 1270 $key = wfRandomString(); 1271 $key2 = wfRandomString(); 1272 $value = wfRandomString(); 1273 1274 $mockWallClock = 1549343530.2053; 1275 $cache->setMockTime( $mockWallClock ); 1276 1277 $calls = 0; 1278 $func = static function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value, &$mockWallClock ) { 1279 ++$calls; 1280 $setOpts['since'] = $mockWallClock; 1281 $mockWallClock += 10; 1282 return $value; 1283 }; 1284 1285 // Value should be given a low logical TTL due to snapshot lag 1286 $curTTL = null; 1287 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1288 $this->assertSame( $value, $ret ); 1289 $this->assertSame( $value, $cache->get( $key, $curTTL ), 'Value was populated' ); 1290 $this->assertEqualsWithDelta( 1.0, $curTTL, 0.01, 'Value has reduced logical TTL' ); 1291 $this->assertSame( 1, $calls, 'Value was generated' ); 1292 1293 $mockWallClock += 2; // low logical TTL expired 1294 1295 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1296 $this->assertSame( $value, $ret ); 1297 $this->assertSame( 2, $calls, 'Callback used (mutex acquired)' ); 1298 1299 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5, 'lowTTL' => -1 ] ); 1300 $this->assertSame( $value, $ret ); 1301 $this->assertSame( 2, $calls, 'Callback was not used (interim value used)' ); 1302 1303 $mockWallClock += 2; // low logical TTL expired 1304 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly 1305 $this->setMutexKey( $bag, $key ); 1306 1307 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1308 $this->assertSame( $value, $ret ); 1309 $this->assertSame( 2, $calls, 'Callback was not used (mutex not acquired)' ); 1310 1311 $mockWallClock += 301; // physical TTL expired 1312 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly 1313 $this->setMutexKey( $bag, $key ); 1314 1315 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1316 $this->assertSame( $value, $ret ); 1317 $this->assertSame( 3, $calls, 'Callback was used (mutex not acquired, not in cache)' ); 1318 1319 $calls = 0; 1320 $func2 = static function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value ) { 1321 ++$calls; 1322 $setOpts['lag'] = 15; 1323 return $value; 1324 }; 1325 1326 // Value should be given a low logical TTL due to replication lag 1327 $curTTL = null; 1328 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1329 $this->assertSame( $value, $ret ); 1330 $this->assertSame( $value, $cache->get( $key2, $curTTL ), 'Value was populated' ); 1331 $this->assertSame( 30.0, $curTTL, 'Value has reduced logical TTL', 0.01 ); 1332 $this->assertSame( 1, $calls, 'Value was generated' ); 1333 1334 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1335 $this->assertSame( $value, $ret ); 1336 $this->assertSame( 1, $calls, 'Callback was used (not expired)' ); 1337 1338 $mockWallClock += 31; 1339 1340 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1341 $this->assertSame( $value, $ret ); 1342 $this->assertSame( 2, $calls, 'Callback was used (mutex acquired)' ); 1343 } 1344 1345 /** 1346 * @covers WANObjectCache::getWithSetCallback() 1347 * @covers WANObjectCache::fetchOrRegenerate() 1348 * @dataProvider provideCoalesceAndMcrouterSettings 1349 */ 1350 public function testBusyValueBasic( array $params ) { 1351 list( $cache, $bag ) = $this->newWanCache( $params ); 1352 $key = wfRandomString(); 1353 $value = wfRandomString(); 1354 $busyValue = wfRandomString(); 1355 1356 $mockWallClock = 1549343530.2053; 1357 $cache->setMockTime( $mockWallClock ); 1358 1359 $calls = 0; 1360 $func = static function () use ( &$calls, $value ) { 1361 ++$calls; 1362 return $value; 1363 }; 1364 1365 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] ); 1366 $this->assertSame( $value, $ret ); 1367 $this->assertSame( 1, $calls, 'Value was populated' ); 1368 1369 $mockWallClock += 0.2; // interim keys not brand new 1370 1371 // Acquire a lock to verify that getWithSetCallback uses busyValue properly 1372 $this->setMutexKey( $bag, $key ); 1373 1374 $checkKeys = [ wfRandomString() ]; // new check keys => force misses 1375 $ret = $cache->getWithSetCallback( $key, 30, $func, 1376 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1377 $this->assertSame( $value, $ret, 'Callback used' ); 1378 $this->assertSame( 2, $calls, 'Callback used' ); 1379 1380 $ret = $cache->getWithSetCallback( $key, 30, $func, 1381 [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1382 $this->assertSame( $value, $ret, 'Old value used' ); 1383 $this->assertSame( 2, $calls, 'Callback was not used' ); 1384 1385 $cache->delete( $key ); // no value at all anymore and still locked 1386 1387 $ret = $cache->getWithSetCallback( $key, 30, $func, 1388 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1389 $this->assertSame( $busyValue, $ret, 'Callback was not used; used busy value' ); 1390 $this->assertSame( 2, $calls, 'Callback was not used; used busy value' ); 1391 1392 $this->clearMutexKey( $bag, $key ); 1393 $mockWallClock += 0.001; // cached values will be newer than tombstone 1394 $ret = $cache->getWithSetCallback( $key, 30, $func, 1395 [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1396 $this->assertSame( $value, $ret, 'Callback was used; saved interim' ); 1397 $this->assertSame( 3, $calls, 'Callback was used; saved interim' ); 1398 1399 $this->setMutexKey( $bag, $key ); 1400 $ret = $cache->getWithSetCallback( $key, 30, $func, 1401 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1402 $this->assertSame( $value, $ret, 'Callback was not used; used interim' ); 1403 $this->assertSame( 3, $calls, 'Callback was not used; used interim' ); 1404 } 1405 1406 public function getBusyValues_Provider() { 1407 $hash = new HashBagOStuff( [] ); 1408 1409 return [ 1410 [ 1411 static function () { 1412 return "Saint Oliver Plunckett"; 1413 }, 1414 'Saint Oliver Plunckett' 1415 ], 1416 [ 'strlen', 'strlen' ], 1417 [ 'WANObjectCache::newEmpty', 'WANObjectCache::newEmpty' ], 1418 [ [ 'WANObjectCache', 'newEmpty' ], [ 'WANObjectCache', 'newEmpty' ] ], 1419 [ [ $hash, 'getLastError' ], [ $hash, 'getLastError' ] ], 1420 [ [ 1, 2, 3 ], [ 1, 2, 3 ] ] 1421 ]; 1422 } 1423 1424 /** 1425 * @covers WANObjectCache::getWithSetCallback() 1426 * @covers WANObjectCache::fetchOrRegenerate() 1427 * @dataProvider getBusyValues_Provider 1428 */ 1429 public function testBusyValueTypes( $busyValue, $expected ) { 1430 list( $cache, $bag ) = $this->newWanCache(); 1431 $key = wfRandomString(); 1432 1433 $mockWallClock = 1549343530.2053; 1434 $cache->setMockTime( $mockWallClock ); 1435 1436 $calls = 0; 1437 $func = static function () use ( &$calls ) { 1438 ++$calls; 1439 return 418; 1440 }; 1441 1442 // Acquire a lock to verify that getWithSetCallback uses busyValue properly 1443 $this->setMutexKey( $bag, $key ); 1444 1445 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] ); 1446 $this->assertSame( $expected, $ret, 'busyValue used as expected' ); 1447 $this->assertSame( 0, $calls, 'busyValue was used' ); 1448 } 1449 1450 /** 1451 * @covers WANObjectCache::getMulti() 1452 */ 1453 public function testGetMulti() { 1454 list( $cache ) = $this->newWanCache(); 1455 1456 $value1 = [ 'this' => 'is', 'a' => 'test' ]; 1457 $value2 = [ 'this' => 'is', 'another' => 'test' ]; 1458 1459 $key1 = wfRandomString(); 1460 $key2 = wfRandomString(); 1461 $key3 = wfRandomString(); 1462 1463 $mockWallClock = 1549343530.2053; 1464 $priorTime = $mockWallClock; // reference time 1465 $cache->setMockTime( $mockWallClock ); 1466 1467 $cache->set( $key1, $value1, 5 ); 1468 $cache->set( $key2, $value2, 10 ); 1469 1470 $curTTLs = []; 1471 $this->assertSame( 1472 [ $key1 => $value1, $key2 => $value2 ], 1473 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs ), 1474 'Result array populated' 1475 ); 1476 1477 $this->assertCount( 2, $curTTLs, "Two current TTLs in array" ); 1478 $this->assertGreaterThan( 0, $curTTLs[$key1], "Key 1 has current TTL > 0" ); 1479 $this->assertGreaterThan( 0, $curTTLs[$key2], "Key 2 has current TTL > 0" ); 1480 1481 $cKey1 = wfRandomString(); 1482 $cKey2 = wfRandomString(); 1483 1484 $mockWallClock += 1; 1485 1486 $curTTLs = []; 1487 $this->assertSame( 1488 [ $key1 => $value1, $key2 => $value2 ], 1489 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ), 1490 "Result array populated even with new check keys" 1491 ); 1492 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1493 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key 1 generated on miss' ); 1494 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1495 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check key 2 generated on miss' ); 1496 $this->assertCount( 2, $curTTLs, "Current TTLs array set" ); 1497 $this->assertLessThanOrEqual( 0, $curTTLs[$key1], 'Key 1 has current TTL <= 0' ); 1498 $this->assertLessThanOrEqual( 0, $curTTLs[$key2], 'Key 2 has current TTL <= 0' ); 1499 1500 $mockWallClock += 1; 1501 1502 $curTTLs = []; 1503 $this->assertSame( 1504 [ $key1 => $value1, $key2 => $value2 ], 1505 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ), 1506 "Result array still populated even with new check keys" 1507 ); 1508 $this->assertCount( 2, $curTTLs, "Current TTLs still array set" ); 1509 $this->assertLessThan( 0, $curTTLs[$key1], 'Key 1 has negative current TTL' ); 1510 $this->assertLessThan( 0, $curTTLs[$key2], 'Key 2 has negative current TTL' ); 1511 } 1512 1513 /** 1514 * @covers WANObjectCache::getMulti() 1515 * @covers WANObjectCache::processCheckKeys() 1516 * @param array $params 1517 * @dataProvider provideCoalesceAndMcrouterSettings 1518 */ 1519 public function testGetMultiCheckKeys( array $params ) { 1520 list( $cache ) = $this->newWanCache( $params ); 1521 1522 $checkAll = wfRandomString(); 1523 $check1 = wfRandomString(); 1524 $check2 = wfRandomString(); 1525 $check3 = wfRandomString(); 1526 $value1 = wfRandomString(); 1527 $value2 = wfRandomString(); 1528 1529 $mockWallClock = 1549343530.2053; 1530 $cache->setMockTime( $mockWallClock ); 1531 1532 // Fake initial check key to be set in the past. Otherwise we'd have to sleep for 1533 // several seconds during the test to assert the behaviour. 1534 foreach ( [ $checkAll, $check1, $check2 ] as $checkKey ) { 1535 $cache->touchCheckKey( $checkKey, WANObjectCache::HOLDOFF_TTL_NONE ); 1536 } 1537 1538 $mockWallClock += 0.100; 1539 1540 $cache->set( 'key1', $value1, 10 ); 1541 $cache->set( 'key2', $value2, 10 ); 1542 1543 $curTTLs = []; 1544 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1545 'key1' => $check1, 1546 $checkAll, 1547 'key2' => $check2, 1548 'key3' => $check3, 1549 ] ); 1550 $this->assertSame( 1551 [ 'key1' => $value1, 'key2' => $value2 ], 1552 $result, 1553 'Initial values' 1554 ); 1555 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key1'], 'Initial ttls' ); 1556 $this->assertLessThanOrEqual( 10.5, $curTTLs['key1'], 'Initial ttls' ); 1557 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key2'], 'Initial ttls' ); 1558 $this->assertLessThanOrEqual( 10.5, $curTTLs['key2'], 'Initial ttls' ); 1559 1560 $mockWallClock += 0.100; 1561 $cache->touchCheckKey( $check1 ); 1562 1563 $curTTLs = []; 1564 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1565 'key1' => $check1, 1566 $checkAll, 1567 'key2' => $check2, 1568 'key3' => $check3, 1569 ] ); 1570 $this->assertSame( 1571 [ 'key1' => $value1, 'key2' => $value2 ], 1572 $result, 1573 'key1 expired by check1, but value still provided' 1574 ); 1575 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 TTL expired' ); 1576 $this->assertGreaterThan( 0, $curTTLs['key2'], 'key2 still valid' ); 1577 1578 $cache->touchCheckKey( $checkAll ); 1579 1580 $curTTLs = []; 1581 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1582 'key1' => $check1, 1583 $checkAll, 1584 'key2' => $check2, 1585 'key3' => $check3, 1586 ] ); 1587 $this->assertSame( 1588 [ 'key1' => $value1, 'key2' => $value2 ], 1589 $result, 1590 'All keys expired by checkAll, but value still provided' 1591 ); 1592 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 expired by checkAll' ); 1593 $this->assertLessThan( 0, $curTTLs['key2'], 'key2 expired by checkAll' ); 1594 } 1595 1596 /** 1597 * @covers WANObjectCache::get() 1598 * @covers WANObjectCache::processCheckKeys() 1599 */ 1600 public function testCheckKeyInitHoldoff() { 1601 list( $cache ) = $this->newWanCache(); 1602 1603 for ( $i = 0; $i < 500; ++$i ) { 1604 $key = wfRandomString(); 1605 $checkKey = wfRandomString(); 1606 // miss, set, hit 1607 $cache->get( $key, $curTTL, [ $checkKey ] ); 1608 $cache->set( $key, 'val', 10 ); 1609 $curTTL = null; 1610 $v = $cache->get( $key, $curTTL, [ $checkKey ] ); 1611 1612 $this->assertSame( 'val', $v ); 1613 $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (miss/set/hit)" ); 1614 } 1615 1616 for ( $i = 0; $i < 500; ++$i ) { 1617 $key = wfRandomString(); 1618 $checkKey = wfRandomString(); 1619 // set, hit 1620 $cache->set( $key, 'val', 10 ); 1621 $curTTL = null; 1622 $v = $cache->get( $key, $curTTL, [ $checkKey ] ); 1623 1624 $this->assertSame( 'val', $v ); 1625 $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (set/hit)" ); 1626 } 1627 } 1628 1629 /** 1630 * @covers WANObjectCache::get() 1631 * @covers WANObjectCache::processCheckKeys() 1632 */ 1633 public function testCheckKeyHoldoff() { 1634 list( $cache ) = $this->newWanCache(); 1635 $key = wfRandomString(); 1636 $checkKey = wfRandomString(); 1637 1638 $mockWallClock = 1549343530.2053; 1639 $cache->setMockTime( $mockWallClock ); 1640 $cache->touchCheckKey( $checkKey, 8 ); 1641 1642 $mockWallClock += 1; 1643 $cache->set( $key, 1, 60 ); 1644 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1645 $this->assertLessThan( 0, $curTTL, "Key in hold-off due to check key" ); 1646 1647 $mockWallClock += 3; 1648 $cache->set( $key, 1, 60 ); 1649 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1650 $this->assertLessThan( 0, $curTTL, "Key in hold-off due to check key" ); 1651 1652 $mockWallClock += 10; 1653 $cache->set( $key, 1, 60 ); 1654 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1655 $this->assertGreaterThan( 0, $curTTL, "Key not in hold-off due to check key" ); 1656 } 1657 1658 /** 1659 * @covers WANObjectCache::delete 1660 * @covers WANObjectCache::relayNonVolatilePurge 1661 * @covers WANObjectCache::relayVolatilePurges 1662 * @covers WANObjectCache::makeTombstonePurgeValue 1663 */ 1664 public function testDelete() { 1665 list( $cache ) = $this->newWanCache(); 1666 $key = wfRandomString(); 1667 $value = wfRandomString(); 1668 $cache->set( $key, $value ); 1669 1670 $curTTL = null; 1671 $v = $cache->get( $key, $curTTL ); 1672 $this->assertSame( $value, $v, "Key was created with value" ); 1673 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); 1674 1675 $cache->delete( $key ); 1676 1677 $curTTL = null; 1678 $v = $cache->get( $key, $curTTL ); 1679 $this->assertSame( false, $v, "Deleted key has false value" ); 1680 $this->assertLessThan( 0, $curTTL, "Deleted key has current TTL < 0" ); 1681 1682 $cache->set( $key, $value . 'more' ); 1683 $v = $cache->get( $key, $curTTL ); 1684 $this->assertSame( false, $v, "Deleted key is tombstoned and has false value" ); 1685 $this->assertLessThan( 0, $curTTL, "Deleted key is tombstoned and has current TTL < 0" ); 1686 1687 $cache->set( $key, $value ); 1688 $cache->delete( $key, WANObjectCache::HOLDOFF_TTL_NONE ); 1689 1690 $curTTL = null; 1691 $v = $cache->get( $key, $curTTL ); 1692 $this->assertSame( false, $v, "Deleted key has false value" ); 1693 $this->assertSame( null, $curTTL, "Deleted key has null current TTL" ); 1694 1695 $cache->set( $key, $value ); 1696 $v = $cache->get( $key, $curTTL ); 1697 $this->assertSame( $value, $v, "Key was created with value" ); 1698 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); 1699 } 1700 1701 /** 1702 * @dataProvider getWithSetCallback_versions_provider 1703 * @covers WANObjectCache::getWithSetCallback() 1704 * @covers WANObjectCache::fetchOrRegenerate() 1705 * @param array $extOpts 1706 * @param bool $versioned 1707 */ 1708 public function testGetWithSetCallback_versions( array $extOpts, $versioned ) { 1709 list( $cache ) = $this->newWanCache(); 1710 1711 $key = wfRandomString(); 1712 $valueV1 = wfRandomString(); 1713 $valueV2 = [ wfRandomString() ]; 1714 1715 $wasSet = 0; 1716 $funcV1 = static function () use ( &$wasSet, $valueV1 ) { 1717 ++$wasSet; 1718 1719 return $valueV1; 1720 }; 1721 1722 $priorValue = false; 1723 $priorAsOf = null; 1724 $funcV2 = static function ( $oldValue, &$ttl, $setOpts, $oldAsOf ) 1725 use ( &$wasSet, $valueV2, &$priorValue, &$priorAsOf ) { 1726 $priorValue = $oldValue; 1727 $priorAsOf = $oldAsOf; 1728 ++$wasSet; 1729 1730 return $valueV2; // new array format 1731 }; 1732 1733 // Set the main key (version N if versioned) 1734 $wasSet = 0; 1735 $v = $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts ); 1736 $this->assertSame( $valueV1, $v, "Value returned" ); 1737 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1738 $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts ); 1739 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 1740 $this->assertSame( $valueV1, $v, "Value not regenerated" ); 1741 1742 if ( $versioned ) { 1743 // Set the key for version N+1 format 1744 $verOpts = [ 'version' => $extOpts['version'] + 1 ]; 1745 } else { 1746 // Start versioning now with the unversioned key still there 1747 $verOpts = [ 'version' => 1 ]; 1748 } 1749 1750 // Value goes to secondary key since V1 already used $key 1751 $wasSet = 0; 1752 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1753 $this->assertSame( $valueV2, $v, "Value returned" ); 1754 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1755 $this->assertSame( false, $priorValue, "Old value not given due to old format" ); 1756 $this->assertSame( null, $priorAsOf, "Old value not given due to old format" ); 1757 1758 $wasSet = 0; 1759 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1760 $this->assertSame( $valueV2, $v, "Value not regenerated (secondary key)" ); 1761 $this->assertSame( 0, $wasSet, "Value not regenerated (secondary key)" ); 1762 1763 // Clear out the older or unversioned key 1764 $cache->delete( $key, 0 ); 1765 1766 // Set the key for next/first versioned format 1767 $wasSet = 0; 1768 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1769 $this->assertSame( $valueV2, $v, "Value returned" ); 1770 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1771 1772 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1773 $this->assertSame( $valueV2, $v, "Value not regenerated (main key)" ); 1774 $this->assertSame( 1, $wasSet, "Value not regenerated (main key)" ); 1775 } 1776 1777 public static function getWithSetCallback_versions_provider() { 1778 return [ 1779 [ [], false ], 1780 [ [ 'version' => 1 ], true ] 1781 ]; 1782 } 1783 1784 /** 1785 * @covers WANObjectCache::useInterimHoldOffCaching 1786 * @covers WANObjectCache::getInterimValue 1787 * @dataProvider provideCoalesceAndMcrouterSettings 1788 */ 1789 public function testInterimHoldOffCaching( array $params ) { 1790 list( $cache, $bag ) = $this->newWanCache( $params ); 1791 1792 $mockWallClock = 1549343530.2053; 1793 $cache->setMockTime( $mockWallClock ); 1794 1795 $value = 'CRL-40-940'; 1796 $wasCalled = 0; 1797 $func = static function () use ( &$wasCalled, $value ) { 1798 $wasCalled++; 1799 1800 return $value; 1801 }; 1802 1803 $cache->useInterimHoldOffCaching( true ); 1804 1805 $key = wfRandomString( 32 ); 1806 $cache->getWithSetCallback( $key, 60, $func ); 1807 $cache->getWithSetCallback( $key, 60, $func ); 1808 $this->assertSame( 1, $wasCalled, 'Value cached' ); 1809 1810 $cache->delete( $key ); // no value at all anymore and still locked 1811 1812 $mockWallClock += 0.001; // cached values will be newer than tombstone 1813 $cache->getWithSetCallback( $key, 60, $func ); 1814 $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim 1815 $cache->getWithSetCallback( $key, 60, $func ); 1816 $this->assertSame( 2, $wasCalled, 'Value interim cached' ); // reuses interim 1817 1818 $mockWallClock += 0.2; // interim key not brand new 1819 $cache->getWithSetCallback( $key, 60, $func ); 1820 $this->assertSame( 3, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim 1821 // Lock up the mutex so interim cache is used 1822 $this->setMutexKey( $bag, $key ); 1823 $cache->getWithSetCallback( $key, 60, $func ); 1824 $this->assertSame( 3, $wasCalled, 'Value interim cached (failed mutex)' ); 1825 $this->clearMutexKey( $bag, $key ); 1826 1827 $cache->useInterimHoldOffCaching( false ); 1828 1829 $wasCalled = 0; 1830 $key = wfRandomString( 32 ); 1831 $cache->getWithSetCallback( $key, 60, $func ); 1832 $cache->getWithSetCallback( $key, 60, $func ); 1833 $this->assertSame( 1, $wasCalled, 'Value cached' ); 1834 1835 $cache->delete( $key ); // no value at all anymore and still locked 1836 1837 $cache->getWithSetCallback( $key, 60, $func ); 1838 $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); 1839 $cache->getWithSetCallback( $key, 60, $func ); 1840 $this->assertSame( 3, $wasCalled, 'Value still regenerated (got mutex)' ); 1841 $cache->getWithSetCallback( $key, 60, $func ); 1842 $this->assertSame( 4, $wasCalled, 'Value still regenerated (got mutex)' ); 1843 // Lock up the mutex so interim cache is used 1844 $this->setMutexKey( $bag, $key ); 1845 $cache->getWithSetCallback( $key, 60, $func ); 1846 $this->assertSame( 5, $wasCalled, 'Value still regenerated (failed mutex)' ); 1847 } 1848 1849 /** 1850 * @covers WANObjectCache::touchCheckKey 1851 * @covers WANObjectCache::resetCheckKey 1852 * @covers WANObjectCache::getCheckKeyTime 1853 * @covers WANObjectCache::getMultiCheckKeyTime 1854 * @covers WANObjectCache::makeCheckPurgeValue 1855 * @covers WANObjectCache::parsePurgeValue 1856 */ 1857 public function testTouchKeys() { 1858 list( $cache ) = $this->newWanCache(); 1859 $key = wfRandomString(); 1860 1861 $mockWallClock = 1549343530.2053; 1862 $priorTime = $mockWallClock; // reference time 1863 $cache->setMockTime( $mockWallClock ); 1864 1865 $mockWallClock += 0.100; 1866 $t0 = $cache->getCheckKeyTime( $key ); 1867 $this->assertGreaterThanOrEqual( $priorTime, $t0, 'Check key auto-created' ); 1868 1869 $priorTime = $mockWallClock; 1870 $mockWallClock += 0.100; 1871 $cache->touchCheckKey( $key ); 1872 $t1 = $cache->getCheckKeyTime( $key ); 1873 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key created' ); 1874 1875 $t2 = $cache->getCheckKeyTime( $key ); 1876 $this->assertSame( $t1, $t2, 'Check key time did not change' ); 1877 1878 $mockWallClock += 0.100; 1879 $cache->touchCheckKey( $key ); 1880 $t3 = $cache->getCheckKeyTime( $key ); 1881 $this->assertGreaterThan( $t2, $t3, 'Check key time increased' ); 1882 1883 $t4 = $cache->getCheckKeyTime( $key ); 1884 $this->assertSame( $t3, $t4, 'Check key time did not change' ); 1885 1886 $mockWallClock += 0.100; 1887 $cache->resetCheckKey( $key ); 1888 $t5 = $cache->getCheckKeyTime( $key ); 1889 $this->assertGreaterThan( $t4, $t5, 'Check key time increased' ); 1890 1891 $t6 = $cache->getCheckKeyTime( $key ); 1892 $this->assertSame( $t5, $t6, 'Check key time did not change' ); 1893 } 1894 1895 /** 1896 * @covers WANObjectCache::getMulti() 1897 * @param array $params 1898 * @dataProvider provideCoalesceAndMcrouterSettings 1899 */ 1900 public function testGetWithSeveralCheckKeys( array $params ) { 1901 list( $cache, $bag ) = $this->newWanCache( $params ); 1902 $key = wfRandomString(); 1903 $tKey1 = wfRandomString(); 1904 $tKey2 = wfRandomString(); 1905 $value = 'meow'; 1906 1907 $mockWallClock = 1549343530.2053; 1908 $priorTime = $mockWallClock; // reference time 1909 $cache->setMockTime( $mockWallClock ); 1910 1911 // Two check keys are newer (given hold-off) than $key, another is older 1912 $this->setCheckKey( $bag, $tKey2, $priorTime - 3 ); 1913 $this->setCheckKey( $bag, $tKey2, $priorTime - 5 ); 1914 $this->setCheckKey( $bag, $tKey1, $priorTime - 30 ); 1915 $cache->set( $key, $value, 30 ); 1916 1917 $curTTL = null; 1918 $v = $cache->get( $key, $curTTL, [ $tKey1, $tKey2 ] ); 1919 $this->assertSame( $value, $v, "Value matches" ); 1920 $this->assertLessThan( -4.9, $curTTL, "Correct CTL" ); 1921 $this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" ); 1922 } 1923 1924 /** 1925 * @covers WANObjectCache::reap() 1926 * @covers WANObjectCache::reapCheckKey() 1927 */ 1928 public function testReap() { 1929 list( $cache, $bag ) = $this->newWanCache(); 1930 $vKey1 = wfRandomString(); 1931 $vKey2 = wfRandomString(); 1932 $tKey1 = wfRandomString(); 1933 $tKey2 = wfRandomString(); 1934 $value = 'moo'; 1935 1936 $knownPurge = time() - 60; 1937 $goodTime = microtime( true ) - 5; 1938 $badTime = microtime( true ) - 300; 1939 1940 $bag->set( 1941 'WANCache:' . $vKey1 . '|#|v', 1942 [ 1943 0 => 1, 1944 1 => $value, 1945 2 => 3600, 1946 3 => $goodTime 1947 ] 1948 ); 1949 $bag->set( 1950 'WANCache:' . $vKey2 . '|#|v', 1951 [ 1952 0 => 1, 1953 1 => $value, 1954 2 => 3600, 1955 3 => $badTime 1956 ] 1957 ); 1958 $bag->set( 1959 'WANCache:' . $tKey1 . '|#|t', 1960 'PURGED:' . $goodTime 1961 ); 1962 $bag->set( 1963 'WANCache:' . $tKey2 . '|#|t', 1964 'PURGED:' . $badTime 1965 ); 1966 1967 $this->assertSame( $value, $cache->get( $vKey1 ) ); 1968 $this->assertSame( $value, $cache->get( $vKey2 ) ); 1969 $cache->reap( $vKey1, $knownPurge, $bad1 ); 1970 $cache->reap( $vKey2, $knownPurge, $bad2 ); 1971 1972 $this->assertSame( false, $bad1 ); 1973 $this->assertTrue( $bad2 ); 1974 1975 $cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 ); 1976 $cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 ); 1977 $this->assertSame( false, $tBad1 ); 1978 $this->assertTrue( $tBad2 ); 1979 } 1980 1981 /** 1982 * @covers WANObjectCache::reap() 1983 */ 1984 public function testReap_fail() { 1985 $backend = $this->getMockBuilder( EmptyBagOStuff::class ) 1986 ->setMethods( [ 'get', 'changeTTL' ] )->getMock(); 1987 $backend->expects( $this->once() )->method( 'get' ) 1988 ->willReturn( [ 1989 0 => 1, 1990 1 => 'value', 1991 2 => 3600, 1992 3 => 300, 1993 ] ); 1994 $backend->expects( $this->once() )->method( 'changeTTL' ) 1995 ->willReturn( false ); 1996 1997 $wanCache = new WANObjectCache( [ 1998 'cache' => $backend 1999 ] ); 2000 2001 $isStale = null; 2002 $ret = $wanCache->reap( 'key', 360, $isStale ); 2003 $this->assertTrue( $isStale, 'value was stale' ); 2004 $this->assertSame( false, $ret, 'changeTTL failed' ); 2005 } 2006 2007 /** 2008 * @covers WANObjectCache::set() 2009 */ 2010 public function testSetWithLag() { 2011 list( $cache ) = $this->newWanCache(); 2012 $now = microtime( true ); 2013 $cache->setMockTime( $now ); 2014 2015 $v = 1; 2016 2017 $key = wfRandomString(); 2018 $opts = [ 'lag' => 300, 'since' => $now, 'walltime' => 0.1 ]; 2019 $cache->set( $key, $v, 30, $opts ); 2020 $this->assertSame( $v, $cache->get( $key ), "Repl-lagged value written." ); 2021 2022 $key = wfRandomString(); 2023 $opts = [ 'lag' => 300, 'since' => $now ]; 2024 $cache->set( $key, $v, 30, $opts ); 2025 $this->assertSame( $v, $cache->get( $key ), "Repl-lagged value written (no walltime)." ); 2026 2027 $key = wfRandomString(); 2028 $opts = [ 'lag' => 0, 'since' => $now - 300, 'walltime' => 0.1 ]; 2029 $cache->set( $key, $v, 30, $opts ); 2030 $this->assertSame( false, $cache->get( $key ), "Trx-lagged value written." ); 2031 2032 $key = wfRandomString(); 2033 $opts = [ 'lag' => 0, 'since' => $now - 300 ]; 2034 $cache->set( $key, $v, 30, $opts ); 2035 $this->assertSame( $v, $cache->get( $key ), "Trx-lagged value written (no walltime)." ); 2036 2037 $key = wfRandomString(); 2038 $opts = [ 'lag' => 5, 'since' => $now - 5, 'walltime' => 0.1 ]; 2039 $cache->set( $key, $v, 30, $opts ); 2040 $this->assertSame( false, $cache->get( $key ), "Trx-lagged value written." ); 2041 2042 $key = wfRandomString(); 2043 $opts = [ 'lag' => 5, 'since' => $now - 5 ]; 2044 $cache->set( $key, $v, 30, $opts ); 2045 $this->assertSame( false, $cache->get( $key ), "Lagged value not written (no walltime)." ); 2046 } 2047 2048 /** 2049 * @covers WANObjectCache::set() 2050 */ 2051 public function testWritePending() { 2052 list( $cache ) = $this->newWanCache(); 2053 $value = 1; 2054 2055 $key = wfRandomString(); 2056 $opts = [ 'pending' => true ]; 2057 $cache->set( $key, $value, 30, $opts ); 2058 $this->assertSame( false, $cache->get( $key ), "Pending value not written." ); 2059 } 2060 2061 public function testMcRouterSupport() { 2062 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2063 ->setMethods( [ 'set', 'delete' ] )->getMock(); 2064 $localBag->expects( $this->never() )->method( 'set' ); 2065 $localBag->expects( $this->never() )->method( 'delete' ); 2066 $wanCache = new WANObjectCache( [ 2067 'cache' => $localBag, 2068 'broadcastRoutingPrefix' => '/*/mw-wan/', 2069 ] ); 2070 $valFunc = static function () { 2071 return 1; 2072 }; 2073 2074 // None of these should use broadcasting commands (e.g. SET, DELETE) 2075 $wanCache->get( 'x' ); 2076 $wanCache->get( 'x', $ctl, [ 'check1' ] ); 2077 $wanCache->getMulti( [ 'x', 'y' ] ); 2078 $wanCache->getMulti( [ 'x', 'y' ], $ctls, [ 'check2' ] ); 2079 $wanCache->getWithSetCallback( 'p', 30, $valFunc ); 2080 $wanCache->getCheckKeyTime( 'zzz' ); 2081 $wanCache->reap( 'x', time() - 300 ); 2082 $wanCache->reap( 'zzz', time() - 300 ); 2083 } 2084 2085 public function testMcRouterSupportBroadcastDelete() { 2086 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2087 ->setMethods( [ 'set' ] )->getMock(); 2088 $wanCache = new WANObjectCache( [ 2089 'cache' => $localBag, 2090 'broadcastRoutingPrefix' => '/*/mw-wan/', 2091 ] ); 2092 2093 $localBag->expects( $this->once() )->method( 'set' ) 2094 ->with( "/*/mw-wan/WANCache:test|#|v" ); 2095 2096 $wanCache->delete( 'test' ); 2097 } 2098 2099 public function testMcRouterSupportBroadcastTouchCK() { 2100 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2101 ->setMethods( [ 'set' ] )->getMock(); 2102 $wanCache = new WANObjectCache( [ 2103 'cache' => $localBag, 2104 'broadcastRoutingPrefix' => '/*/mw-wan/', 2105 ] ); 2106 2107 $localBag->expects( $this->once() )->method( 'set' ) 2108 ->with( "/*/mw-wan/WANCache:test|#|t" ); 2109 2110 $wanCache->touchCheckKey( 'test' ); 2111 } 2112 2113 public function testMcRouterSupportBroadcastResetCK() { 2114 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2115 ->setMethods( [ 'delete' ] )->getMock(); 2116 $wanCache = new WANObjectCache( [ 2117 'cache' => $localBag, 2118 'broadcastRoutingPrefix' => '/*/mw-wan/', 2119 ] ); 2120 2121 $localBag->expects( $this->once() )->method( 'delete' ) 2122 ->with( "/*/mw-wan/WANCache:test|#|t" ); 2123 2124 $wanCache->resetCheckKey( 'test' ); 2125 } 2126 2127 public function testEpoch() { 2128 $bag = new HashBagOStuff(); 2129 $cache = new WANObjectCache( [ 'cache' => $bag ] ); 2130 $key = $cache->makeGlobalKey( 'The whole of the Law' ); 2131 2132 $now = microtime( true ); 2133 $cache->setMockTime( $now ); 2134 2135 $cache->set( $key, 'Do what thou Wilt' ); 2136 $cache->touchCheckKey( $key ); 2137 2138 $then = $now; 2139 $now += 30; 2140 $this->assertSame( 'Do what thou Wilt', $cache->get( $key ) ); 2141 $this->assertEqualsWithDelta( $then, $cache->getCheckKeyTime( $key ), 0.01, 'Check key init' ); 2142 2143 $cache = new WANObjectCache( [ 2144 'cache' => $bag, 2145 'epoch' => $now - 3600 2146 ] ); 2147 $cache->setMockTime( $now ); 2148 2149 $this->assertSame( 'Do what thou Wilt', $cache->get( $key ) ); 2150 $this->assertEqualsWithDelta( $then, $cache->getCheckKeyTime( $key ), 0.01, 'Check key kept' ); 2151 2152 $now += 30; 2153 $cache = new WANObjectCache( [ 2154 'cache' => $bag, 2155 'epoch' => $now + 3600 2156 ] ); 2157 $cache->setMockTime( $now ); 2158 2159 $this->assertSame( false, $cache->get( $key ), 'Key rejected due to epoch' ); 2160 $this->assertEqualsWithDelta( $now, $cache->getCheckKeyTime( $key ), 0.01, 'Check key reset' ); 2161 } 2162 2163 /** 2164 * @dataProvider provideAdaptiveTTL 2165 * @covers WANObjectCache::adaptiveTTL() 2166 * @param float|int $ago 2167 * @param int $maxTTL 2168 * @param int $minTTL 2169 * @param float $factor 2170 * @param int $adaptiveTTL 2171 */ 2172 public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) { 2173 list( $cache ) = $this->newWanCache(); 2174 $mtime = $ago ? time() - $ago : $ago; 2175 $margin = 5; 2176 $ttl = $cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor ); 2177 2178 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); 2179 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); 2180 2181 $ttl = $cache->adaptiveTTL( (string)$mtime, $maxTTL, $minTTL, $factor ); 2182 2183 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); 2184 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); 2185 } 2186 2187 public static function provideAdaptiveTTL() { 2188 return [ 2189 [ 3600, 900, 30, 0.2, 720 ], 2190 [ 3600, 500, 30, 0.2, 500 ], 2191 [ 3600, 86400, 800, 0.2, 800 ], 2192 [ false, 86400, 800, 0.2, 800 ], 2193 [ null, 86400, 800, 0.2, 800 ] 2194 ]; 2195 } 2196 2197 /** 2198 * @covers WANObjectCache::__construct 2199 * @covers WANObjectCache::newEmpty 2200 */ 2201 public function testNewEmpty() { 2202 $this->assertInstanceOf( 2203 WANObjectCache::class, 2204 WANObjectCache::newEmpty() 2205 ); 2206 } 2207 2208 /** 2209 * @covers WANObjectCache::setLogger 2210 */ 2211 public function testSetLogger() { 2212 list( $cache ) = $this->newWanCache(); 2213 $this->assertSame( null, $cache->setLogger( new Psr\Log\NullLogger ) ); 2214 } 2215 2216 /** 2217 * @covers WANObjectCache::getQoS 2218 */ 2219 public function testGetQoS() { 2220 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2221 ->setMethods( [ 'getQoS' ] )->getMock(); 2222 $backend->expects( $this->once() )->method( 'getQoS' ) 2223 ->willReturn( BagOStuff::QOS_UNKNOWN ); 2224 $wanCache = new WANObjectCache( [ 'cache' => $backend ] ); 2225 2226 $this->assertSame( 2227 $wanCache::QOS_UNKNOWN, 2228 $wanCache->getQoS( $wanCache::ATTR_EMULATION ) 2229 ); 2230 } 2231 2232 /** 2233 * @covers WANObjectCache::makeKey 2234 */ 2235 public function testMakeKey() { 2236 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2237 ->setMethods( [ 'makeKey' ] )->getMock(); 2238 $backend->expects( $this->once() )->method( 'makeKey' ) 2239 ->willReturn( 'special' ); 2240 2241 $wanCache = new WANObjectCache( [ 2242 'cache' => $backend 2243 ] ); 2244 2245 $this->assertSame( 'special', $wanCache->makeKey( 'a', 'b' ) ); 2246 } 2247 2248 /** 2249 * @covers WANObjectCache::makeGlobalKey 2250 */ 2251 public function testMakeGlobalKey() { 2252 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2253 ->setMethods( [ 'makeGlobalKey' ] )->getMock(); 2254 $backend->expects( $this->once() )->method( 'makeGlobalKey' ) 2255 ->willReturn( 'special' ); 2256 2257 $wanCache = new WANObjectCache( [ 2258 'cache' => $backend 2259 ] ); 2260 2261 $this->assertSame( 'special', $wanCache->makeGlobalKey( 'a', 'b' ) ); 2262 } 2263 2264 public static function statsKeyProvider() { 2265 return [ 2266 [ 'domain:page:5', 'page' ], 2267 [ 'domain:main-key', 'main-key' ], 2268 [ 'domain:page:history', 'page' ], 2269 // Regression test for T232907 2270 [ 'domain:foo-bar-1.2:abc:v2', 'foo-bar-1_2' ], 2271 [ 'missingdomainkey', 'missingdomainkey' ] 2272 ]; 2273 } 2274 2275 /** 2276 * @dataProvider statsKeyProvider 2277 * @covers WANObjectCache::determineKeyClassForStats 2278 * @param string $key 2279 * @param string $class 2280 */ 2281 public function testStatsKeyClass( $key, $class ) { 2282 /** @var WANObjectCache $wanCache */ 2283 $wanCache = TestingAccessWrapper::newFromObject( new WANObjectCache( [ 2284 'cache' => new HashBagOStuff 2285 ] ) ); 2286 2287 $this->assertSame( $class, $wanCache->determineKeyClassForStats( $key ) ); 2288 } 2289 2290 /** 2291 * @covers WANObjectCache::makeMultiKeys 2292 */ 2293 public function testMakeMultiKeys() { 2294 list( $cache ) = $this->newWanCache(); 2295 2296 $ids = [ 1, 2, 3, 4, 4, 5, 6, 6, 7, 7 ]; 2297 $keyCallback = static function ( $id, WANObjectCache $cache ) { 2298 return $cache->makeKey( 'key', $id ); 2299 }; 2300 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2301 2302 $expected = [ 2303 "local:key:1" => 1, 2304 "local:key:2" => 2, 2305 "local:key:3" => 3, 2306 "local:key:4" => 4, 2307 "local:key:5" => 5, 2308 "local:key:6" => 6, 2309 "local:key:7" => 7 2310 ]; 2311 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2312 2313 $ids = [ '1', '2', '3', '4', '4', '5', '6', '6', '7', '7' ]; 2314 $keyCallback = static function ( $id, WANObjectCache $cache ) { 2315 return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' ); 2316 }; 2317 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2318 2319 $expected = [ 2320 "global:key:1:a:1:b" => '1', 2321 "global:key:2:a:2:b" => '2', 2322 "global:key:3:a:3:b" => '3', 2323 "global:key:4:a:4:b" => '4', 2324 "global:key:5:a:5:b" => '5', 2325 "global:key:6:a:6:b" => '6', 2326 "global:key:7:a:7:b" => '7' 2327 ]; 2328 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2329 } 2330 2331 /** 2332 * @covers WANObjectCache::makeMultiKeys 2333 */ 2334 public function testMakeMultiKeysIntString() { 2335 list( $cache ) = $this->newWanCache(); 2336 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7, '7' ]; 2337 $keyCallback = static function ( $id, WANObjectCache $cache ) { 2338 return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' ); 2339 }; 2340 2341 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2342 2343 $expected = [ 2344 "global:key:1:a:1:b" => 1, 2345 "global:key:2:a:2:b" => 2, 2346 "global:key:3:a:3:b" => 3, 2347 "global:key:4:a:4:b" => 4, 2348 "global:key:5:a:5:b" => 5, 2349 "global:key:6:a:6:b" => 6, 2350 "global:key:7:a:7:b" => 7 2351 ]; 2352 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2353 } 2354 2355 /** 2356 * @covers WANObjectCache::makeMultiKeys 2357 */ 2358 public function testMakeMultiKeysCollision() { 2359 list( $cache ) = $this->newWanCache(); 2360 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7 ]; 2361 2362 $this->expectException( UnexpectedValueException::class ); 2363 $cache->makeMultiKeys( 2364 $ids, 2365 static function ( $id ) { 2366 return "keymod:" . $id % 3; 2367 } 2368 ); 2369 } 2370 2371 /** 2372 * @covers WANObjectCache::multiRemap 2373 */ 2374 public function testMultiRemap() { 2375 list( $cache ) = $this->newWanCache(); 2376 $a = [ 'a', 'b', 'c' ]; 2377 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3 ]; 2378 2379 $this->assertSame( 2380 [ 'a' => 1, 'b' => 2, 'c' => 3 ], 2381 $cache->multiRemap( $a, $res ) 2382 ); 2383 2384 $a = [ 'a', 'b', 'c', 'c', 'd' ]; 2385 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3, 'keyD' => 4 ]; 2386 2387 $this->assertSame( 2388 [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ], 2389 $cache->multiRemap( $a, $res ) 2390 ); 2391 } 2392 2393 /** 2394 * @covers WANObjectCache::hash256 2395 */ 2396 public function testHash256() { 2397 list( $cache ) = $this->newWanCache( [ 'epoch' => 5 ] ); 2398 $this->assertEquals( 2399 'f402bce76bfa1136adc705d8d5719911ce1fe61f0ad82ddf79a15f3c4de6ec4c', 2400 $cache->hash256( 'x' ) 2401 ); 2402 2403 list( $cache ) = $this->newWanCache( [ 'epoch' => 50 ] ); 2404 $this->assertSame( 2405 'f79a126722f0a682c4c500509f1b61e836e56c4803f92edc89fc281da5caa54e', 2406 $cache->hash256( 'x' ) 2407 ); 2408 2409 list( $cache ) = $this->newWanCache( [ 'secret' => 'garden' ] ); 2410 $this->assertSame( 2411 '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', 2412 $cache->hash256( 'x' ) 2413 ); 2414 2415 list( $cache ) = $this->newWanCache( [ 'secret' => 'garden', 'epoch' => 3 ] ); 2416 $this->assertSame( 2417 '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', 2418 $cache->hash256( 'x' ) 2419 ); 2420 } 2421 2422 /** 2423 * @covers WANObjectCache::getWithSetCallback() 2424 * @covers WANObjectCache::fetchOrRegenerate() 2425 * @covers WANObjectCache::get() 2426 * @covers WANObjectCache::set() 2427 * @dataProvider provideCoalesceAndMcrouterSettings 2428 * @param array $params 2429 * @param string|null $keyNeedle 2430 */ 2431 public function testCoalesceKeys( array $params, $keyNeedle ) { 2432 list( $cache, $bag ) = $this->newWanCache( $params ); 2433 $key = wfRandomString(); 2434 $callback = static function () { 2435 return 2020; 2436 }; 2437 2438 $cache->getWithSetCallback( $key, 60, $callback ); 2439 $wrapper = TestingAccessWrapper::newFromObject( $bag ); 2440 foreach ( array_keys( $wrapper->bag ) as $bagKey ) { 2441 if ( $keyNeedle === null ) { 2442 $this->assertNotRegExp( '/[#{}]/', $bagKey, 'Respects "coalesceKeys"' ); 2443 } else { 2444 $this->assertStringContainsString( 2445 $keyNeedle, 2446 $bagKey, 2447 'Respects "coalesceKeys"' 2448 ); 2449 } 2450 } 2451 } 2452 2453 /** 2454 * @param string $key 2455 * @param string $expectedCollection 2456 * @covers WANObjectCache::getCollectionFromSisterKey() 2457 * @dataProvider provideCollectionKeys 2458 */ 2459 public function testgetCollectionFromSisterKey( $key, $expectedCollection ) { 2460 $this->assertSame( 2461 $expectedCollection, 2462 WANObjectCache::getCollectionFromSisterKey( $key ), 2463 'Correct key collection name' 2464 ); 2465 } 2466 2467 public static function provideCollectionKeys() { 2468 return [ 2469 [ 'WANCache:collection:a:b|#|v', 'collection' ], 2470 [ 'WANCache:{collection:a:b}:v', 'collection' ], 2471 [ 'WANCache:collection:a:b|#|t', 'internal' ], 2472 [ 'WANCache:{collection:a:b}:t', 'internal' ], 2473 [ 'WANCache:improper-key', 'internal' ], 2474 ]; 2475 } 2476} 2477 2478class McrouterHashBagOStuff extends HashBagOStuff { 2479 public function set( $key, $value, $exptime = 0, $flags = 0 ) { 2480 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::set() calls 2481 // https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup 2482 if ( preg_match( '#^/\*/[^/]+/(.*)$#', $key, $m ) ) { 2483 $key = $m[1]; 2484 } 2485 2486 return parent::set( $key, $value, $exptime, $flags ); 2487 } 2488 2489 public function delete( $key, $flags = 0 ) { 2490 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls 2491 // https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup 2492 if ( preg_match( '#^/\*/[^/]+/(.*)$#', $key, $m ) ) { 2493 $key = $m[1]; 2494 } 2495 2496 return parent::delete( $key, $flags ); 2497 } 2498} 2499 2500class NearExpiringWANObjectCache extends WANObjectCache { 2501 private const CLOCK_SKEW = 1; 2502 2503 protected function worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) { 2504 return ( $curTTL > 0 && ( $curTTL + self::CLOCK_SKEW ) < $lowTTL ); 2505 } 2506} 2507 2508class PopularityRefreshingWANObjectCache extends WANObjectCache { 2509 protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) { 2510 return ( ( $now - $asOf ) > $timeTillRefresh ); 2511 } 2512} 2513