1<?php /** @noinspection PhpStaticAsDynamicMethodCallInspection */ 2 3use Wikimedia\TestingAccessWrapper; 4 5/** 6 * @covers WANObjectCache::wrap 7 * @covers WANObjectCache::unwrap 8 * @covers WANObjectCache::worthRefreshExpiring 9 * @covers WANObjectCache::worthRefreshPopular 10 * @covers WANObjectCache::isValid 11 * @covers WANObjectCache::getWarmupKeyMisses 12 * @covers WANObjectCache::makeSisterKey 13 * @covers WANObjectCache::makeSisterKeys 14 * @covers WANObjectCache::getProcessCache 15 * @covers WANObjectCache::getNonProcessCachedMultiKeys 16 * @covers WANObjectCache::getRawKeysForWarmup 17 * @covers WANObjectCache::getInterimValue 18 * @covers WANObjectCache::setInterimValue 19 */ 20class WANObjectCacheTest extends PHPUnit\Framework\TestCase { 21 22 use MediaWikiCoversValidator; 23 24 /** 25 * @param array $params [optional] 26 * @return WANObjectCache[]|HashBagOStuff[] (WANObjectCache, BagOStuff) 27 */ 28 private function newWanCache( array $params = [] ) { 29 if ( !empty( $params['mcrouterAware'] ) ) { 30 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls 31 $bag = new McrouterHashBagOStuff(); 32 } else { 33 $bag = new HashBagOStuff(); 34 } 35 36 $cache = new WANObjectCache( [ 'cache' => $bag ] + $params ); 37 38 return [ $cache, $bag ]; 39 } 40 41 /** 42 * @dataProvider provideSetAndGet 43 * @covers WANObjectCache::set() 44 * @covers WANObjectCache::get() 45 * @covers WANObjectCache::makeKey() 46 * @param mixed $value 47 * @param int $ttl 48 */ 49 public function testSetAndGet( $value, $ttl ) { 50 list( $cache ) = $this->newWanCache(); 51 52 $curTTL = null; 53 $asOf = null; 54 $key = $cache->makeKey( 'x', wfRandomString() ); 55 56 $cache->get( $key, $curTTL, [], $asOf ); 57 $this->assertSame( null, $curTTL, "Current TTL (absent)" ); 58 $this->assertSame( null, $asOf, "Current as-of-time (absent)" ); 59 60 $t = microtime( true ); 61 62 $cache->set( $key, $value, $cache::TTL_UNCACHEABLE ); 63 $cache->get( $key, $curTTL, [], $asOf ); 64 $this->assertSame( null, $curTTL, "Current TTL (TTL_UNCACHEABLE)" ); 65 $this->assertSame( null, $asOf, "Current as-of-time (TTL_UNCACHEABLE)" ); 66 67 $cache->set( $key, $value, $ttl ); 68 69 $this->assertSame( $value, $cache->get( $key, $curTTL, [], $asOf ) ); 70 if ( $ttl === INF ) { 71 $this->assertSame( INF, $curTTL, "Current TTL" ); 72 } else { 73 $this->assertGreaterThan( 0, $curTTL, "Current TTL" ); 74 $this->assertLessThanOrEqual( $ttl, $curTTL, "Current TTL < nominal TTL" ); 75 } 76 $this->assertGreaterThanOrEqual( $t - 1, $asOf, "As-of-time in range of set() time" ); 77 $this->assertLessThanOrEqual( $t + 1, $asOf, "As-of-time in range of set() time" ); 78 } 79 80 public static function provideSetAndGet() { 81 return [ 82 // value, ttl 83 [ 14141, 3 ], 84 [ 3535.666, 3 ], 85 [ [], 3 ], 86 [ '0', 3 ], 87 [ (object)[ 'meow' ], 3 ], 88 [ INF, 3 ], 89 [ '', 3 ], 90 [ 'pizzacat', INF ], 91 [ null, 80 ] 92 ]; 93 } 94 95 /** 96 * @covers WANObjectCache::get() 97 * @covers WANObjectCache::makeGlobalKey() 98 */ 99 public function testGetNotExists() { 100 list( $cache ) = $this->newWanCache(); 101 102 $key = $cache->makeGlobalKey( 'y', wfRandomString(), 'p' ); 103 $curTTL = null; 104 $value = $cache->get( $key, $curTTL ); 105 106 $this->assertSame( false, $value, "Return value" ); 107 $this->assertSame( null, $curTTL, "current TTL" ); 108 } 109 110 /** 111 * @covers WANObjectCache::set() 112 */ 113 public function testSetOver() { 114 list( $cache ) = $this->newWanCache(); 115 116 $key = wfRandomString(); 117 for ( $i = 0; $i < 3; ++$i ) { 118 $value = wfRandomString(); 119 $cache->set( $key, $value, 3 ); 120 121 $this->assertSame( $cache->get( $key ), $value ); 122 } 123 } 124 125 public static function provideStaleSetParams() { 126 return [ 127 // Given a db transaction (trx lag) that started 30s ago, 128 // we generally don't want to cache its values. 129 [ 30, 0.0, false ], 130 [ 30, 2, false ], 131 [ 30, 10, false ], 132 [ 30, 20, false ], 133 // If the main reason we've hit 30s is that we spent 134 // a lot of time in the regeneration callback (as opposed 135 // to time mainly having passed before the cache computation) 136 // then cache it for at least a little while. 137 [ 30, 28, true ], 138 // Also if we don't know, cache it for a little while. 139 [ 30, null, true ], 140 ]; 141 } 142 143 /** 144 * @covers WANObjectCache::set() 145 * @dataProvider provideStaleSetParams 146 * @param int $ago 147 * @param float|null $walltime 148 * @param bool $cacheable 149 */ 150 public function testStaleSet( $ago, $walltime, $cacheable ) { 151 list( $cache ) = $this->newWanCache(); 152 $mockWallClock = 1549343530.2053; 153 $cache->setMockTime( $mockWallClock ); 154 155 $key = wfRandomString(); 156 $value = wfRandomString(); 157 158 $cache->set( 159 $key, 160 $value, 161 $cache::TTL_MINUTE, 162 [ 'since' => $mockWallClock - $ago, 'walltime' => $walltime ] 163 ); 164 165 $this->assertSame( 166 $cacheable ? $value : false, 167 $cache->get( $key ), 168 "Stale set() value ignored" 169 ); 170 } 171 172 /** 173 * @covers WANObjectCache::getWithSetCallback 174 */ 175 public function testProcessCacheTTL() { 176 list( $cache ) = $this->newWanCache(); 177 $mockWallClock = 1549343530.2053; 178 $cache->setMockTime( $mockWallClock ); 179 180 $key = "mykey-" . wfRandomString(); 181 182 $hits = 0; 183 $callback = function ( $oldValue, &$ttl, &$setOpts ) use ( &$hits ) { 184 ++$hits; 185 return 42; 186 }; 187 188 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 189 $cache->delete( $key, $cache::HOLDOFF_NONE ); // clear persistent cache 190 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 191 $this->assertSame( 1, $hits, "Value process cached" ); 192 193 $mockWallClock += 6; 194 $cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] ); 195 $this->assertSame( 2, $hits, "Value expired in process cache" ); 196 } 197 198 /** 199 * @covers WANObjectCache::getWithSetCallback 200 */ 201 public function testProcessCacheLruAndDelete() { 202 list( $cache ) = $this->newWanCache(); 203 $mockWallClock = 1549343530.2053; 204 $cache->setMockTime( $mockWallClock ); 205 206 $hit = 0; 207 $fn = function () use ( &$hit ) { 208 ++$hit; 209 return 42; 210 }; 211 $keysA = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 212 $keysB = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 213 $pcg = [ 'thiscache:1', 'thatcache:1', 'somecache:1' ]; 214 215 foreach ( $keysA as $i => $key ) { 216 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 217 } 218 $this->assertSame( 3, $hit, "Values not cached yet" ); 219 220 foreach ( $keysA as $i => $key ) { 221 // Should not evict from process cache 222 $cache->delete( $key ); 223 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 224 } 225 $this->assertSame( 3, $hit, "Values cached; not cleared by delete()" ); 226 227 foreach ( $keysB as $i => $key ) { 228 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 229 } 230 $this->assertSame( 6, $hit, "New values not cached yet" ); 231 232 foreach ( $keysB as $i => $key ) { 233 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 234 } 235 $this->assertSame( 6, $hit, "New values cached" ); 236 237 foreach ( $keysA as $i => $key ) { 238 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 239 } 240 $this->assertSame( 9, $hit, "Prior values evicted by new values" ); 241 } 242 243 /** 244 * @covers WANObjectCache::getWithSetCallback 245 */ 246 public function testProcessCacheInterimKeys() { 247 list( $cache ) = $this->newWanCache(); 248 $mockWallClock = 1549343530.2053; 249 $cache->setMockTime( $mockWallClock ); 250 251 $hit = 0; 252 $fn = function () use ( &$hit ) { 253 ++$hit; 254 return 42; 255 }; 256 $keysA = [ wfRandomString(), wfRandomString(), wfRandomString() ]; 257 $pcg = [ 'thiscache:1', 'thatcache:1', 'somecache:1' ]; 258 259 foreach ( $keysA as $i => $key ) { 260 $cache->delete( $key ); // tombstone key 261 $mockWallClock += 0.001; // cached values will be newer than tombstone 262 // Get into process cache (specific group) and interim cache 263 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5, 'pcGroup' => $pcg[$i] ] ); 264 } 265 $this->assertSame( 3, $hit ); 266 267 // Get into process cache (default group) 268 $key = reset( $keysA ); 269 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 270 $this->assertSame( 3, $hit, "Value recently interim-cached" ); 271 272 $mockWallClock += 0.2; // interim key not brand new 273 $cache->clearProcessCache(); 274 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 275 $this->assertSame( 4, $hit, "Value calculated (interim key not recent and reset)" ); 276 $cache->getWithSetCallback( $key, 100, $fn, [ 'pcTTL' => 5 ] ); 277 $this->assertSame( 4, $hit, "Value process cached" ); 278 } 279 280 /** 281 * @covers WANObjectCache::getWithSetCallback 282 */ 283 public function testProcessCacheNesting() { 284 list( $cache ) = $this->newWanCache(); 285 $mockWallClock = 1549343530.2053; 286 $cache->setMockTime( $mockWallClock ); 287 288 $keyOuter = "outer-" . wfRandomString(); 289 $keyInner = "inner-" . wfRandomString(); 290 291 $innerHit = 0; 292 $innerFn = function () use ( &$innerHit ) { 293 ++$innerHit; 294 return 42; 295 }; 296 297 $outerHit = 0; 298 $outerFn = function () use ( $keyInner, $innerFn, $cache, &$outerHit ) { 299 ++$outerHit; 300 $v = $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 301 302 return 43 + $v; 303 }; 304 305 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 306 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 307 308 $this->assertSame( 1, $innerHit, "Inner callback value cached" ); 309 $cache->delete( $keyInner, $cache::HOLDOFF_NONE ); 310 $mockWallClock += 1; 311 312 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 313 $this->assertSame( 1, $innerHit, "Inner callback process cached" ); 314 315 // Outer key misses and inner key process cache value is refused 316 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 317 318 $this->assertSame( 1, $outerHit, "Outer callback value not yet cached" ); 319 $this->assertSame( 2, $innerHit, "Inner callback value process cache skipped" ); 320 321 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 322 323 $this->assertSame( 1, $outerHit, "Outer callback value cached" ); 324 325 $cache->delete( $keyInner, $cache::HOLDOFF_NONE ); 326 $cache->delete( $keyOuter, $cache::HOLDOFF_NONE ); 327 $mockWallClock += 1; 328 $cache->clearProcessCache(); 329 $cache->getWithSetCallback( $keyOuter, 100, $outerFn ); 330 331 $this->assertSame( 2, $outerHit, "Outer callback value not yet cached" ); 332 $this->assertSame( 3, $innerHit, "Inner callback value not yet cached" ); 333 334 $cache->delete( $keyInner, $cache::HOLDOFF_NONE ); 335 $mockWallClock += 1; 336 $cache->getWithSetCallback( $keyInner, 100, $innerFn, [ 'pcTTL' => 5 ] ); 337 338 $this->assertSame( 3, $innerHit, "Inner callback value process cached" ); 339 } 340 341 /** 342 * @dataProvider getWithSetCallback_provider 343 * @covers WANObjectCache::getWithSetCallback() 344 * @covers WANObjectCache::fetchOrRegenerate() 345 * @param array $extOpts 346 */ 347 public function testGetWithSetCallback( array $extOpts ) { 348 list( $cache ) = $this->newWanCache(); 349 350 $key = wfRandomString(); 351 $value = wfRandomString(); 352 $cKey1 = wfRandomString(); 353 $cKey2 = wfRandomString(); 354 355 $priorValue = null; 356 $priorAsOf = null; 357 $wasSet = 0; 358 $func = function ( $old, &$ttl, &$opts, $asOf ) 359 use ( &$wasSet, &$priorValue, &$priorAsOf, $value ) { 360 ++$wasSet; 361 $priorValue = $old; 362 $priorAsOf = $asOf; 363 $ttl = 20; // override with another value 364 return $value; 365 }; 366 367 $mockWallClock = 1549343530.2053; 368 $priorTime = $mockWallClock; // reference time 369 $cache->setMockTime( $mockWallClock ); 370 371 $wasSet = 0; 372 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] + $extOpts ); 373 $this->assertSame( $value, $v, "Value returned" ); 374 $this->assertSame( 1, $wasSet, "Value regenerated" ); 375 $this->assertSame( false, $priorValue, "No prior value" ); 376 $this->assertSame( null, $priorAsOf, "No prior value" ); 377 378 $curTTL = null; 379 $cache->get( $key, $curTTL ); 380 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 381 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 382 383 $wasSet = 0; 384 $v = $cache->getWithSetCallback( 385 $key, 30, $func, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 386 $this->assertSame( $value, $v, "Value returned" ); 387 $this->assertSame( 0, $wasSet, "Value not regenerated" ); 388 389 $mockWallClock += 1; 390 391 $wasSet = 0; 392 $v = $cache->getWithSetCallback( 393 $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 394 ); 395 $this->assertSame( $value, $v, "Value returned" ); 396 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 397 $this->assertSame( $value, $priorValue, "Has prior value" ); 398 $this->assertIsFloat( $priorAsOf, "Has prior value" ); 399 $t1 = $cache->getCheckKeyTime( $cKey1 ); 400 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 401 $t2 = $cache->getCheckKeyTime( $cKey2 ); 402 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 403 404 $mockWallClock += 0.2; // interim key is not brand new and check keys have past values 405 $priorTime = $mockWallClock; // reference time 406 $wasSet = 0; 407 $v = $cache->getWithSetCallback( 408 $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 409 ); 410 $this->assertSame( $value, $v, "Value returned" ); 411 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 412 $t1 = $cache->getCheckKeyTime( $cKey1 ); 413 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 414 $t2 = $cache->getCheckKeyTime( $cKey2 ); 415 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 416 417 $curTTL = null; 418 $v = $cache->get( $key, $curTTL, [ $cKey1, $cKey2 ] ); 419 $this->assertSame( $value, $v, "Value returned" ); 420 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 421 422 $wasSet = 0; 423 $key = wfRandomString(); 424 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts ); 425 $this->assertSame( $value, $v, "Value returned" ); 426 $cache->delete( $key ); 427 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts ); 428 $this->assertSame( $value, $v, "Value still returned after deleted" ); 429 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 430 431 $oldValReceived = -1; 432 $oldAsOfReceived = -1; 433 $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf ) 434 use ( &$oldValReceived, &$oldAsOfReceived, &$wasSet ) { 435 ++$wasSet; 436 $oldValReceived = $oldVal; 437 $oldAsOfReceived = $oldAsOf; 438 439 return 'xxx' . $wasSet; 440 }; 441 442 $mockWallClock = 1549343530.2053; 443 $priorTime = $mockWallClock; // reference time 444 445 $wasSet = 0; 446 $key = wfRandomString(); 447 $v = $cache->getWithSetCallback( 448 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 449 $this->assertSame( 'xxx1', $v, "Value returned" ); 450 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 451 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 452 453 $mockWallClock += 40; 454 $v = $cache->getWithSetCallback( 455 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 456 $this->assertSame( 'xxx2', $v, "Value still returned after expired" ); 457 $this->assertSame( 2, $wasSet, "Value recalculated while expired" ); 458 $this->assertSame( 'xxx1', $oldValReceived, "Callback got stale value" ); 459 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got stale value" ); 460 461 $mockWallClock += 260; 462 $v = $cache->getWithSetCallback( 463 $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts ); 464 $this->assertSame( 'xxx3', $v, "Value still returned after expired" ); 465 $this->assertSame( 3, $wasSet, "Value recalculated while expired" ); 466 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 467 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 468 469 $mockWallClock = ( $priorTime - $cache::HOLDOFF_TTL - 1 ); 470 $wasSet = 0; 471 $key = wfRandomString(); 472 $checkKey = $cache->makeKey( 'template', 'X' ); 473 $cache->touchCheckKey( $checkKey ); // init check key 474 $mockWallClock = $priorTime; 475 $v = $cache->getWithSetCallback( 476 $key, 477 $cache::TTL_INDEFINITE, 478 $checkFunc, 479 [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts 480 ); 481 $this->assertSame( 'xxx1', $v, "Value returned" ); 482 $this->assertSame( 1, $wasSet, "Value computed" ); 483 $this->assertSame( false, $oldValReceived, "Callback got no stale value" ); 484 $this->assertSame( null, $oldAsOfReceived, "Callback got no stale value" ); 485 486 $mockWallClock += $cache::TTL_HOUR; // some time passes 487 $v = $cache->getWithSetCallback( 488 $key, 489 $cache::TTL_INDEFINITE, 490 $checkFunc, 491 [ 492 'graceTTL' => $cache::TTL_WEEK, 493 'checkKeys' => [ $checkKey ], 494 'ageNew' => -1 495 ] + $extOpts 496 ); 497 $this->assertSame( 'xxx1', $v, "Cached value returned" ); 498 $this->assertSame( 1, $wasSet, "Cached value returned" ); 499 500 $cache->touchCheckKey( $checkKey ); // make key stale 501 $mockWallClock += 0.01; // ~1 week left of grace (barely stale to avoid refreshes) 502 503 $v = $cache->getWithSetCallback( 504 $key, 505 $cache::TTL_INDEFINITE, 506 $checkFunc, 507 [ 508 'graceTTL' => $cache::TTL_WEEK, 509 'checkKeys' => [ $checkKey ], 510 'ageNew' => -1, 511 ] + $extOpts 512 ); 513 $this->assertSame( 'xxx1', $v, "Value still returned after expired (in grace)" ); 514 $this->assertSame( 1, $wasSet, "Value still returned after expired (in grace)" ); 515 516 // Chance of refresh increase to unity as staleness approaches graceTTL 517 $mockWallClock += $cache::TTL_WEEK; // 8 days of being stale 518 $v = $cache->getWithSetCallback( 519 $key, 520 $cache::TTL_INDEFINITE, 521 $checkFunc, 522 [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts 523 ); 524 $this->assertSame( 'xxx2', $v, "Value was recomputed (past grace)" ); 525 $this->assertSame( 2, $wasSet, "Value was recomputed (past grace)" ); 526 $this->assertSame( 'xxx1', $oldValReceived, "Callback got post-grace stale value" ); 527 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got post-grace stale value" ); 528 } 529 530 /** 531 * @dataProvider getWithSetCallback_provider 532 * @covers WANObjectCache::getWithSetCallback() 533 * @covers WANObjectCache::fetchOrRegenerate() 534 * @param array $extOpts 535 */ 536 public function testGetWithSetCallback_touched( array $extOpts ) { 537 list( $cache ) = $this->newWanCache(); 538 539 $mockWallClock = 1549343530.2053; 540 $cache->setMockTime( $mockWallClock ); 541 542 $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf ) 543 use ( &$wasSet ) { 544 ++$wasSet; 545 546 return 'xxx' . $wasSet; 547 }; 548 549 $key = wfRandomString(); 550 $wasSet = 0; 551 $touched = null; 552 $touchedCallback = function () use ( &$touched ) { 553 return $touched; 554 }; 555 $v = $cache->getWithSetCallback( 556 $key, 557 $cache::TTL_INDEFINITE, 558 $checkFunc, 559 [ 'touchedCallback' => $touchedCallback ] + $extOpts 560 ); 561 $mockWallClock += 60; 562 $v = $cache->getWithSetCallback( 563 $key, 564 $cache::TTL_INDEFINITE, 565 $checkFunc, 566 [ 'touchedCallback' => $touchedCallback ] + $extOpts 567 ); 568 $this->assertSame( 'xxx1', $v, "Value was computed once" ); 569 $this->assertSame( 1, $wasSet, "Value was computed once" ); 570 571 $touched = $mockWallClock - 10; 572 $v = $cache->getWithSetCallback( 573 $key, 574 $cache::TTL_INDEFINITE, 575 $checkFunc, 576 [ 'touchedCallback' => $touchedCallback ] + $extOpts 577 ); 578 $v = $cache->getWithSetCallback( 579 $key, 580 $cache::TTL_INDEFINITE, 581 $checkFunc, 582 [ 'touchedCallback' => $touchedCallback ] + $extOpts 583 ); 584 $this->assertSame( 'xxx2', $v, "Value was recomputed once" ); 585 $this->assertSame( 2, $wasSet, "Value was recomputed once" ); 586 } 587 588 public static function getWithSetCallback_provider() { 589 return [ 590 [ [], false ], 591 [ [ 'version' => 1 ], true ] 592 ]; 593 } 594 595 public function testPreemtiveRefresh() { 596 $value = 'KatCafe'; 597 $wasSet = 0; 598 $func = function ( $old, &$ttl, &$opts, $asOf ) use ( &$wasSet, &$value ) 599 { 600 ++$wasSet; 601 return $value; 602 }; 603 604 $cache = new NearExpiringWANObjectCache( [ 'cache' => new HashBagOStuff() ] ); 605 $mockWallClock = 1549343530.2053; 606 $cache->setMockTime( $mockWallClock ); 607 608 $wasSet = 0; 609 $key = wfRandomString(); 610 $opts = [ 'lowTTL' => 30 ]; 611 $v = $cache->getWithSetCallback( $key, 20, $func, $opts ); 612 $this->assertSame( $value, $v, "Value returned" ); 613 $this->assertSame( 1, $wasSet, "Value calculated" ); 614 615 $mockWallClock += 0.2; // interim key is not brand new 616 $v = $cache->getWithSetCallback( $key, 20, $func, $opts ); 617 $this->assertSame( 2, $wasSet, "Value re-calculated" ); 618 619 $wasSet = 0; 620 $key = wfRandomString(); 621 $opts = [ 'lowTTL' => 1 ]; 622 $v = $cache->getWithSetCallback( $key, 30, $func, $opts ); 623 $this->assertSame( $value, $v, "Value returned" ); 624 $this->assertSame( 1, $wasSet, "Value calculated" ); 625 $v = $cache->getWithSetCallback( $key, 30, $func, $opts ); 626 $this->assertSame( 1, $wasSet, "Value cached" ); 627 628 $asycList = []; 629 $asyncHandler = function ( $callback ) use ( &$asycList ) { 630 $asycList[] = $callback; 631 }; 632 $cache = new NearExpiringWANObjectCache( [ 633 'cache' => new HashBagOStuff(), 634 'asyncHandler' => $asyncHandler 635 ] ); 636 637 $mockWallClock = 1549343530.2053; 638 $priorTime = $mockWallClock; // reference time 639 $cache->setMockTime( $mockWallClock ); 640 641 $wasSet = 0; 642 $key = wfRandomString(); 643 $opts = [ 'lowTTL' => 100 ]; 644 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 645 $this->assertSame( $value, $v, "Value returned" ); 646 $this->assertSame( 1, $wasSet, "Value calculated" ); 647 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 648 $this->assertSame( 1, $wasSet, "Cached value used" ); 649 $this->assertSame( $v, $value, "Value cached" ); 650 651 $mockWallClock += 250; 652 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 653 $this->assertSame( $value, $v, "Value returned" ); 654 $this->assertSame( 1, $wasSet, "Stale value used" ); 655 $this->assertCount( 1, $asycList, "Refresh deferred." ); 656 $value = 'NewCatsInTown'; // change callback return value 657 $asycList[0](); // run the refresh callback 658 $asycList = []; 659 $this->assertSame( 2, $wasSet, "Value calculated at later time" ); 660 $this->assertSame( [], $asycList, "No deferred refreshes added." ); 661 $v = $cache->getWithSetCallback( $key, 300, $func, $opts ); 662 $this->assertSame( $value, $v, "New value stored" ); 663 664 $cache = new PopularityRefreshingWANObjectCache( [ 665 'cache' => new HashBagOStuff() 666 ] ); 667 668 $mockWallClock = $priorTime; 669 $cache->setMockTime( $mockWallClock ); 670 671 $wasSet = 0; 672 $key = wfRandomString(); 673 $opts = [ 'hotTTR' => 900 ]; 674 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 675 $this->assertSame( $value, $v, "Value returned" ); 676 $this->assertSame( 1, $wasSet, "Value calculated" ); 677 678 $mockWallClock += 30; 679 680 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 681 $this->assertSame( 1, $wasSet, "Value cached" ); 682 683 $mockWallClock = $priorTime; 684 $wasSet = 0; 685 $key = wfRandomString(); 686 $opts = [ 'hotTTR' => 10 ]; 687 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 688 $this->assertSame( $value, $v, "Value returned" ); 689 $this->assertSame( 1, $wasSet, "Value calculated" ); 690 691 $mockWallClock += 30; 692 693 $v = $cache->getWithSetCallback( $key, 60, $func, $opts ); 694 $this->assertSame( $value, $v, "Value returned" ); 695 $this->assertSame( 2, $wasSet, "Value re-calculated" ); 696 } 697 698 /** 699 * @dataProvider getMultiWithSetCallback_provider 700 * @covers WANObjectCache::getMultiWithSetCallback 701 * @covers WANObjectCache::makeMultiKeys 702 * @covers WANObjectCache::getMulti 703 * @param array $extOpts 704 */ 705 public function testGetMultiWithSetCallback( array $extOpts ) { 706 list( $cache ) = $this->newWanCache(); 707 708 $keyA = wfRandomString(); 709 $keyB = wfRandomString(); 710 $keyC = wfRandomString(); 711 $cKey1 = wfRandomString(); 712 $cKey2 = wfRandomString(); 713 714 $priorValue = null; 715 $priorAsOf = null; 716 $wasSet = 0; 717 $genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use ( 718 &$wasSet, &$priorValue, &$priorAsOf 719 ) { 720 ++$wasSet; 721 $priorValue = $old; 722 $priorAsOf = $asOf; 723 $ttl = 20; // override with another value 724 return "@$id$"; 725 }; 726 727 $mockWallClock = 1549343530.2053; 728 $priorTime = $mockWallClock; // reference time 729 $cache->setMockTime( $mockWallClock ); 730 731 $wasSet = 0; 732 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); 733 $value = "@3353$"; 734 $v = $cache->getMultiWithSetCallback( 735 $keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts ); 736 $this->assertSame( $value, $v[$keyA], "Value returned" ); 737 $this->assertSame( 1, $wasSet, "Value regenerated" ); 738 $this->assertSame( false, $priorValue, "No prior value" ); 739 $this->assertSame( null, $priorAsOf, "No prior value" ); 740 741 $curTTL = null; 742 $cache->get( $keyA, $curTTL ); 743 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 744 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 745 746 $wasSet = 0; 747 $value = "@efef$"; 748 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 749 $v = $cache->getMultiWithSetCallback( 750 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 751 $this->assertSame( $value, $v[$keyB], "Value returned" ); 752 $this->assertSame( 1, $wasSet, "Value regenerated" ); 753 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 754 755 $v = $cache->getMultiWithSetCallback( 756 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts ); 757 $this->assertSame( $value, $v[$keyB], "Value returned" ); 758 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 759 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 760 761 $mockWallClock += 1; 762 763 $wasSet = 0; 764 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 765 $v = $cache->getMultiWithSetCallback( 766 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 767 ); 768 $this->assertSame( $value, $v[$keyB], "Value returned" ); 769 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 770 $this->assertSame( $value, $priorValue, "Has prior value" ); 771 $this->assertIsFloat( $priorAsOf, "Has prior value" ); 772 $t1 = $cache->getCheckKeyTime( $cKey1 ); 773 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 774 $t2 = $cache->getCheckKeyTime( $cKey2 ); 775 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 776 777 $mockWallClock += 0.01; 778 $priorTime = $mockWallClock; 779 $value = "@43636$"; 780 $wasSet = 0; 781 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); 782 $v = $cache->getMultiWithSetCallback( 783 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 784 ); 785 $this->assertSame( $value, $v[$keyC], "Value returned" ); 786 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 787 $t1 = $cache->getCheckKeyTime( $cKey1 ); 788 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 789 $t2 = $cache->getCheckKeyTime( $cKey2 ); 790 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 791 792 $curTTL = null; 793 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); 794 $this->assertSame( $value, $v, "Value returned" ); 795 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 796 797 $wasSet = 0; 798 $key = wfRandomString(); 799 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 800 $v = $cache->getMultiWithSetCallback( 801 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 802 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); 803 $cache->delete( $key ); 804 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 805 $v = $cache->getMultiWithSetCallback( 806 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 807 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); 808 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 809 810 $calls = 0; 811 $ids = [ 1, 2, 3, 4, 5, 6 ]; 812 $keyFunc = function ( $id, WANObjectCache $wanCache ) { 813 return $wanCache->makeKey( 'test', $id ); 814 }; 815 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); 816 $genFunc = function ( $id, $oldValue, &$ttl, array &$setops ) use ( &$calls ) { 817 ++$calls; 818 819 return "val-{$id}"; 820 }; 821 $values = $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); 822 823 $this->assertSame( 824 [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], 825 array_values( $values ), 826 "Correct values in correct order" 827 ); 828 $this->assertSame( 829 array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ), 830 array_keys( $values ), 831 "Correct keys in correct order" 832 ); 833 $this->assertSame( count( $ids ), $calls ); 834 835 $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); 836 $this->assertSame( count( $ids ), $calls, "Values cached" ); 837 838 // Mock the BagOStuff to assure only one getMulti() call given process caching 839 $localBag = $this->getMockBuilder( HashBagOStuff::class ) 840 ->setMethods( [ 'getMulti' ] )->getMock(); 841 $localBag->expects( $this->exactly( 1 ) )->method( 'getMulti' )->willReturn( [ 842 'WANCache:v:' . 'k1' => 'val-id1', 843 'WANCache:v:' . 'k2' => 'val-id2' 844 ] ); 845 $wanCache = new WANObjectCache( [ 'cache' => $localBag ] ); 846 847 // Warm the process cache 848 $keyedIds = new ArrayIterator( [ 'k1' => 'id1', 'k2' => 'id2' ] ); 849 $this->assertSame( 850 [ 'k1' => 'val-id1', 'k2' => 'val-id2' ], 851 $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] ) 852 ); 853 // Use the process cache 854 $this->assertSame( 855 [ 'k1' => 'val-id1', 'k2' => 'val-id2' ], 856 $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] ) 857 ); 858 } 859 860 public static function getMultiWithSetCallback_provider() { 861 return [ 862 [ [], false ], 863 [ [ 'version' => 1 ], true ] 864 ]; 865 } 866 867 /** 868 * @dataProvider getMultiWithSetCallbackRefresh_provider 869 * @param bool $expiring 870 * @param bool $popular 871 * @param array $idsByKey 872 */ 873 public function testGetMultiWithSetCallbackRefresh( $expiring, $popular, array $idsByKey ) { 874 $deferredCbs = []; 875 $bag = new HashBagOStuff(); 876 $cache = $this->getMockBuilder( WANObjectCache::class ) 877 ->setMethods( [ 'worthRefreshExpiring', 'worthRefreshPopular' ] ) 878 ->setConstructorArgs( [ 879 [ 880 'cache' => $bag, 881 'asyncHandler' => function ( $callback ) use ( &$deferredCbs ) { 882 $deferredCbs[] = $callback; 883 } 884 ] 885 ] ) 886 ->getMock(); 887 888 $cache->method( 'worthRefreshExpiring' )->willReturn( $expiring ); 889 $cache->method( 'worthRefreshPopular' )->willReturn( $popular ); 890 891 $wasSet = 0; 892 $keyedIds = new ArrayIterator( $idsByKey ); 893 $genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use ( &$wasSet ) { 894 ++$wasSet; 895 $ttl = 20; // override with another value 896 return "@$id$"; 897 }; 898 899 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 900 $this->assertSame( count( $idsByKey ), $wasSet, "Initial sets" ); 901 $this->assertSame( [], $deferredCbs, "No deferred callbacks yet" ); 902 foreach ( $idsByKey as $key => $id ) { 903 $this->assertSame( "@$id$", $v[$key], "Initial cache value generation" ); 904 } 905 906 $wasSet = 0; 907 $preemptiveRefresh = ( $expiring || $popular ); 908 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 909 $this->assertSame( 0, $wasSet, "No values generated" ); 910 $this->assertSame( 911 $preemptiveRefresh ? count( $idsByKey ) : 0, 912 count( $deferredCbs ), 913 "Deferred callbacks queued" 914 ); 915 foreach ( $idsByKey as $key => $id ) { 916 $this->assertSame( "@$id$", $v[$key], "Cached value reused; refresh scheduled" ); 917 } 918 919 // Run the deferred callbacks... 920 $deferredCbsReady = $deferredCbs; 921 $deferredCbs = []; // empty by-reference queue 922 foreach ( $deferredCbsReady as $deferredCb ) { 923 $deferredCb(); 924 } 925 926 $this->assertSame( 927 ( $preemptiveRefresh ? count( $idsByKey ) : 0 ), 928 $wasSet, 929 "Deferred callback regenerations" 930 ); 931 $this->assertSame( [], $deferredCbs, "Deferred callbacks queue empty" ); 932 933 $wasSet = 0; 934 $v = $cache->getMultiWithSetCallback( $keyedIds, 30, $genFunc ); 935 $this->assertSame( 936 0, 937 $wasSet, 938 "Deferred callbacks did not run again" 939 ); 940 foreach ( $idsByKey as $key => $id ) { 941 $this->assertSame( "@$id$", $v[$key], "Cached value OK after deferred refresh run" ); 942 } 943 } 944 945 public static function getMultiWithSetCallbackRefresh_provider() { 946 return [ 947 [ true, true, [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ] ], 948 [ true, false, [ 'a' => 'x', 'b' => 'y', 'c' => 'z', 'd' => 'w' ] ], 949 [ false, true, [ 'a' => 'p', 'b' => 'q', 'c' => 'r', 'd' => 's' ] ], 950 [ false, false, [ 'a' => '%', 'b' => '^', 'c' => '&', 'd' => 'ç' ] ] 951 ]; 952 } 953 954 /** 955 * @dataProvider getMultiWithUnionSetCallback_provider 956 * @covers WANObjectCache::getMultiWithUnionSetCallback() 957 * @covers WANObjectCache::makeMultiKeys() 958 * @param array $extOpts 959 */ 960 public function testGetMultiWithUnionSetCallback( array $extOpts ) { 961 list( $cache ) = $this->newWanCache(); 962 963 $keyA = wfRandomString(); 964 $keyB = wfRandomString(); 965 $keyC = wfRandomString(); 966 $cKey1 = wfRandomString(); 967 $cKey2 = wfRandomString(); 968 969 $wasSet = 0; 970 $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( 971 &$wasSet, &$priorValue, &$priorAsOf 972 ) { 973 $newValues = []; 974 foreach ( $ids as $id ) { 975 ++$wasSet; 976 $newValues[$id] = "@$id$"; 977 $ttls[$id] = 20; // override with another value 978 } 979 980 return $newValues; 981 }; 982 983 $mockWallClock = 1549343530.2053; 984 $priorTime = $mockWallClock; // reference time 985 $cache->setMockTime( $mockWallClock ); 986 987 $wasSet = 0; 988 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); 989 $value = "@3353$"; 990 $v = $cache->getMultiWithUnionSetCallback( 991 $keyedIds, 30, $genFunc, $extOpts ); 992 $this->assertSame( $value, $v[$keyA], "Value returned" ); 993 $this->assertSame( 1, $wasSet, "Value regenerated" ); 994 995 $curTTL = null; 996 $cache->get( $keyA, $curTTL ); 997 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); 998 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); 999 1000 $wasSet = 0; 1001 $value = "@efef$"; 1002 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 1003 $v = $cache->getMultiWithUnionSetCallback( 1004 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); 1005 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1006 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1007 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 1008 1009 $v = $cache->getMultiWithUnionSetCallback( 1010 $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); 1011 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1012 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 1013 $this->assertSame( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" ); 1014 1015 $mockWallClock += 1; 1016 1017 $wasSet = 0; 1018 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); 1019 $v = $cache->getMultiWithUnionSetCallback( 1020 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 1021 ); 1022 $this->assertSame( $value, $v[$keyB], "Value returned" ); 1023 $this->assertSame( 1, $wasSet, "Value regenerated due to check keys" ); 1024 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1025 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); 1026 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1027 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); 1028 1029 $mockWallClock += 0.01; 1030 $priorTime = $mockWallClock; 1031 $value = "@43636$"; 1032 $wasSet = 0; 1033 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); 1034 $v = $cache->getMultiWithUnionSetCallback( 1035 $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts 1036 ); 1037 $this->assertSame( $value, $v[$keyC], "Value returned" ); 1038 $this->assertSame( 1, $wasSet, "Value regenerated due to still-recent check keys" ); 1039 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1040 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); 1041 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1042 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); 1043 1044 $curTTL = null; 1045 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); 1046 $this->assertSame( $value, $v, "Value returned" ); 1047 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); 1048 1049 $wasSet = 0; 1050 $key = wfRandomString(); 1051 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 1052 $v = $cache->getMultiWithUnionSetCallback( 1053 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 1054 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); 1055 $cache->delete( $key ); 1056 $keyedIds = new ArrayIterator( [ $key => 242424 ] ); 1057 $v = $cache->getMultiWithUnionSetCallback( 1058 $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); 1059 $this->assertSame( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); 1060 $this->assertSame( 1, $wasSet, "Value process cached while deleted" ); 1061 1062 $calls = 0; 1063 $ids = [ 1, 2, 3, 4, 5, 6 ]; 1064 $keyFunc = function ( $id, WANObjectCache $wanCache ) { 1065 return $wanCache->makeKey( 'test', $id ); 1066 }; 1067 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); 1068 $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( &$calls ) { 1069 $newValues = []; 1070 foreach ( $ids as $id ) { 1071 ++$calls; 1072 $newValues[$id] = "val-{$id}"; 1073 } 1074 1075 return $newValues; 1076 }; 1077 $values = $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); 1078 1079 $this->assertSame( 1080 [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], 1081 array_values( $values ), 1082 "Correct values in correct order" 1083 ); 1084 $this->assertSame( 1085 array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $cache ) ), 1086 array_keys( $values ), 1087 "Correct keys in correct order" 1088 ); 1089 $this->assertSame( count( $ids ), $calls ); 1090 1091 $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); 1092 $this->assertSame( count( $ids ), $calls, "Values cached" ); 1093 } 1094 1095 public static function getMultiWithUnionSetCallback_provider() { 1096 return [ 1097 [ [], false ], 1098 [ [ 'version' => 1 ], true ] 1099 ]; 1100 } 1101 1102 public static function provideCoalesceAndMcrouterSettings() { 1103 return [ 1104 [ [ 'mcrouterAware' => false, 'coalesceKeys' => false ], null ], 1105 [ [ 'mcrouterAware' => false, 'coalesceKeys' => true ], '{' ], 1106 [ [ 'mcrouterAware' => true, 'cluster' => 'test', 'coalesceKeys' => false ], null ], 1107 [ [ 'mcrouterAware' => true, 'cluster' => 'test', 'coalesceKeys' => true ], '|#|' ] 1108 ]; 1109 } 1110 1111 /** 1112 * @dataProvider getMultiWithUnionSetCallbackRefresh_provider 1113 * @param bool $expiring 1114 * @param bool $popular 1115 * @param array $idsByKey 1116 */ 1117 public function testGetMultiWithUnionSetCallbackRefresh( $expiring, $popular, array $idsByKey ) { 1118 $deferredCbs = []; 1119 $bag = new HashBagOStuff(); 1120 $cache = $this->getMockBuilder( WANObjectCache::class ) 1121 ->setMethods( [ 'worthRefreshExpiring', 'worthRefreshPopular' ] ) 1122 ->setConstructorArgs( [ 1123 [ 1124 'cache' => $bag, 1125 'asyncHandler' => function ( $callback ) use ( &$deferredCbs ) { 1126 $deferredCbs[] = $callback; 1127 } 1128 ] 1129 ] ) 1130 ->getMock(); 1131 1132 $cache->expects( $this->any() )->method( 'worthRefreshExpiring' )->willReturn( $expiring ); 1133 $cache->expects( $this->any() )->method( 'worthRefreshPopular' )->willReturn( $popular ); 1134 1135 $wasSet = 0; 1136 $keyedIds = new ArrayIterator( $idsByKey ); 1137 $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( &$wasSet ) { 1138 $newValues = []; 1139 foreach ( $ids as $id ) { 1140 ++$wasSet; 1141 $newValues[$id] = "@$id$"; 1142 $ttls[$id] = 20; // override with another value 1143 } 1144 1145 return $newValues; 1146 }; 1147 1148 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1149 $this->assertSame( count( $idsByKey ), $wasSet, "Initial sets" ); 1150 $this->assertSame( [], $deferredCbs, "No deferred callbacks yet" ); 1151 foreach ( $idsByKey as $key => $id ) { 1152 $this->assertSame( "@$id$", $v[$key], "Initial cache value generation" ); 1153 } 1154 1155 $preemptiveRefresh = ( $expiring || $popular ); 1156 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1157 $this->assertSame( count( $idsByKey ), $wasSet, "Deferred callbacks did not run yet" ); 1158 $this->assertSame( 1159 $preemptiveRefresh ? count( $idsByKey ) : 0, 1160 count( $deferredCbs ), 1161 "Deferred callbacks queued" 1162 ); 1163 foreach ( $idsByKey as $key => $id ) { 1164 $this->assertSame( "@$id$", $v[$key], "Cached value reused; refresh scheduled" ); 1165 } 1166 1167 // Run the deferred callbacks... 1168 $deferredCbsReady = $deferredCbs; 1169 $deferredCbs = []; // empty by-reference queue 1170 foreach ( $deferredCbsReady as $deferredCb ) { 1171 $deferredCb(); 1172 } 1173 1174 $this->assertSame( 1175 count( $idsByKey ) * ( $preemptiveRefresh ? 2 : 1 ), 1176 $wasSet, 1177 "Deferred callback regenerations" 1178 ); 1179 $this->assertSame( [], $deferredCbs, "Deferred callbacks queue empty" ); 1180 1181 $v = $cache->getMultiWithUnionSetCallback( $keyedIds, 30, $genFunc ); 1182 $this->assertSame( 1183 count( $idsByKey ) * ( $preemptiveRefresh ? 2 : 1 ), 1184 $wasSet, 1185 "Deferred callbacks did not run again yet" 1186 ); 1187 foreach ( $idsByKey as $key => $id ) { 1188 $this->assertSame( "@$id$", $v[$key], "Cached value OK after deferred refresh run" ); 1189 } 1190 } 1191 1192 public static function getMultiWithUnionSetCallbackRefresh_provider() { 1193 return [ 1194 [ true, true, [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ] ], 1195 [ true, false, [ 'a' => 'x', 'b' => 'y', 'c' => 'z', 'd' => 'w' ] ], 1196 [ false, true, [ 'a' => 'p', 'b' => 'q', 'c' => 'r', 'd' => 's' ] ], 1197 [ false, false, [ 'a' => '%', 'b' => '^', 'c' => '&', 'd' => 'ç' ] ] 1198 ]; 1199 } 1200 1201 /** 1202 * @covers WANObjectCache::getWithSetCallback() 1203 * @covers WANObjectCache::fetchOrRegenerate() 1204 * @dataProvider provideCoalesceAndMcrouterSettings 1205 * @param array $params 1206 */ 1207 public function testLockTSE( array $params ) { 1208 list( $cache, $bag ) = $this->newWanCache( $params ); 1209 $key = wfRandomString(); 1210 $value = wfRandomString(); 1211 1212 $mockWallClock = 1549343530.2053; 1213 $cache->setMockTime( $mockWallClock ); 1214 1215 $calls = 0; 1216 $func = function () use ( &$calls, $value, $cache, $key ) { 1217 ++$calls; 1218 return $value; 1219 }; 1220 1221 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] ); 1222 $this->assertSame( $value, $ret ); 1223 $this->assertSame( 1, $calls, 'Value was populated' ); 1224 1225 // Acquire the mutex to verify that getWithSetCallback uses lockTSE properly 1226 $this->setMutexKey( $bag, $key ); 1227 1228 $checkKeys = [ wfRandomString() ]; // new check keys => force misses 1229 $ret = $cache->getWithSetCallback( $key, 30, $func, 1230 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1231 $this->assertSame( $value, $ret, 'Old value used' ); 1232 $this->assertSame( 1, $calls, 'Callback was not used' ); 1233 1234 $cache->delete( $key ); // no value at all anymore and still locked 1235 1236 $mockWallClock += 0.001; // cached values will be newer than tombstone 1237 $ret = $cache->getWithSetCallback( $key, 30, $func, 1238 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1239 $this->assertSame( $value, $ret, 'Callback was used; interim saved' ); 1240 $this->assertSame( 2, $calls, 'Callback was used; interim saved' ); 1241 1242 $ret = $cache->getWithSetCallback( $key, 30, $func, 1243 [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] ); 1244 $this->assertSame( $value, $ret, 'Callback was not used; used interim (mutex failed)' ); 1245 $this->assertSame( 2, $calls, 'Callback was not used; used interim (mutex failed)' ); 1246 } 1247 1248 private function setMutexKey( BagOStuff $bag, $key ) { 1249 // Cover all formats for "coalesceKeys"/"mcrouterAware" 1250 $bag->add( "WANCache:$key|#|m", 1 ); 1251 $bag->add( "WANCache:{" . $key . "}:m", 1 ); 1252 $bag->add( "WANCache:m:$key", 1 ); 1253 } 1254 1255 private function clearMutexKey( BagOStuff $bag, $key ) { 1256 // Cover all formats for "coalesceKeys"/"mcrouterAware" 1257 $bag->delete( "WANCache:$key|#|m" ); 1258 $bag->delete( "WANCache:{" . $key . "}:m" ); 1259 $bag->delete( "WANCache:m:$key" ); 1260 } 1261 1262 private function setCheckKey( BagOStuff $bag, $key, $time ) { 1263 $bag->set( "WANCache:$key|#|t", "PURGED:$time" ); 1264 $bag->set( "WANCache:{" . $key . "}:t", "PURGED:$time" ); 1265 $bag->set( "WANCache:t:$key", "PURGED:$time" ); 1266 } 1267 1268 /** 1269 * @covers WANObjectCache::getWithSetCallback() 1270 * @covers WANObjectCache::fetchOrRegenerate() 1271 * @covers WANObjectCache::set() 1272 * @dataProvider provideCoalesceAndMcrouterSettings 1273 * @param array $params 1274 */ 1275 public function testLockTSESlow( array $params ) { 1276 list( $cache, $bag ) = $this->newWanCache( $params ); 1277 $key = wfRandomString(); 1278 $key2 = wfRandomString(); 1279 $value = wfRandomString(); 1280 1281 $mockWallClock = 1549343530.2053; 1282 $cache->setMockTime( $mockWallClock ); 1283 1284 $calls = 0; 1285 $func = function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value, &$mockWallClock ) { 1286 ++$calls; 1287 $setOpts['since'] = $mockWallClock; 1288 $mockWallClock += 10; 1289 return $value; 1290 }; 1291 1292 // Value should be given a low logical TTL due to snapshot lag 1293 $curTTL = null; 1294 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1295 $this->assertSame( $value, $ret ); 1296 $this->assertSame( $value, $cache->get( $key, $curTTL ), 'Value was populated' ); 1297 $this->assertEqualsWithDelta( 1.0, $curTTL, 0.01, 'Value has reduced logical TTL' ); 1298 $this->assertSame( 1, $calls, 'Value was generated' ); 1299 1300 $mockWallClock += 2; // low logical TTL expired 1301 1302 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1303 $this->assertSame( $value, $ret ); 1304 $this->assertSame( 2, $calls, 'Callback used (mutex acquired)' ); 1305 1306 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5, 'lowTTL' => -1 ] ); 1307 $this->assertSame( $value, $ret ); 1308 $this->assertSame( 2, $calls, 'Callback was not used (interim value used)' ); 1309 1310 $mockWallClock += 2; // low logical TTL expired 1311 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly 1312 $this->setMutexKey( $bag, $key ); 1313 1314 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1315 $this->assertSame( $value, $ret ); 1316 $this->assertSame( 2, $calls, 'Callback was not used (mutex not acquired)' ); 1317 1318 $mockWallClock += 301; // physical TTL expired 1319 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly 1320 $this->setMutexKey( $bag, $key ); 1321 1322 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] ); 1323 $this->assertSame( $value, $ret ); 1324 $this->assertSame( 3, $calls, 'Callback was used (mutex not acquired, not in cache)' ); 1325 1326 $calls = 0; 1327 $func2 = function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value ) { 1328 ++$calls; 1329 $setOpts['lag'] = 15; 1330 return $value; 1331 }; 1332 1333 // Value should be given a low logical TTL due to replication lag 1334 $curTTL = null; 1335 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1336 $this->assertSame( $value, $ret ); 1337 $this->assertSame( $value, $cache->get( $key2, $curTTL ), 'Value was populated' ); 1338 $this->assertSame( 30.0, $curTTL, 'Value has reduced logical TTL', 0.01 ); 1339 $this->assertSame( 1, $calls, 'Value was generated' ); 1340 1341 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1342 $this->assertSame( $value, $ret ); 1343 $this->assertSame( 1, $calls, 'Callback was used (not expired)' ); 1344 1345 $mockWallClock += 31; 1346 1347 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] ); 1348 $this->assertSame( $value, $ret ); 1349 $this->assertSame( 2, $calls, 'Callback was used (mutex acquired)' ); 1350 } 1351 1352 /** 1353 * @covers WANObjectCache::getWithSetCallback() 1354 * @covers WANObjectCache::fetchOrRegenerate() 1355 * @dataProvider provideCoalesceAndMcrouterSettings 1356 * @param array $params 1357 */ 1358 public function testBusyValueBasic( array $params ) { 1359 list( $cache, $bag ) = $this->newWanCache( $params ); 1360 $key = wfRandomString(); 1361 $value = wfRandomString(); 1362 $busyValue = wfRandomString(); 1363 1364 $mockWallClock = 1549343530.2053; 1365 $cache->setMockTime( $mockWallClock ); 1366 1367 $calls = 0; 1368 $func = function () use ( &$calls, $value ) { 1369 ++$calls; 1370 return $value; 1371 }; 1372 1373 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] ); 1374 $this->assertSame( $value, $ret ); 1375 $this->assertSame( 1, $calls, 'Value was populated' ); 1376 1377 $mockWallClock += 0.2; // interim keys not brand new 1378 1379 // Acquire a lock to verify that getWithSetCallback uses busyValue properly 1380 $this->setMutexKey( $bag, $key ); 1381 1382 $checkKeys = [ wfRandomString() ]; // new check keys => force misses 1383 $ret = $cache->getWithSetCallback( $key, 30, $func, 1384 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1385 $this->assertSame( $value, $ret, 'Callback used' ); 1386 $this->assertSame( 2, $calls, 'Callback used' ); 1387 1388 $ret = $cache->getWithSetCallback( $key, 30, $func, 1389 [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1390 $this->assertSame( $value, $ret, 'Old value used' ); 1391 $this->assertSame( 2, $calls, 'Callback was not used' ); 1392 1393 $cache->delete( $key ); // no value at all anymore and still locked 1394 1395 $ret = $cache->getWithSetCallback( $key, 30, $func, 1396 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1397 $this->assertSame( $busyValue, $ret, 'Callback was not used; used busy value' ); 1398 $this->assertSame( 2, $calls, 'Callback was not used; used busy value' ); 1399 1400 $this->clearMutexKey( $bag, $key ); 1401 $mockWallClock += 0.001; // cached values will be newer than tombstone 1402 $ret = $cache->getWithSetCallback( $key, 30, $func, 1403 [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1404 $this->assertSame( $value, $ret, 'Callback was used; saved interim' ); 1405 $this->assertSame( 3, $calls, 'Callback was used; saved interim' ); 1406 1407 $this->setMutexKey( $bag, $key ); 1408 $ret = $cache->getWithSetCallback( $key, 30, $func, 1409 [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] ); 1410 $this->assertSame( $value, $ret, 'Callback was not used; used interim' ); 1411 $this->assertSame( 3, $calls, 'Callback was not used; used interim' ); 1412 } 1413 1414 public function getBusyValues_Provider() { 1415 $hash = new HashBagOStuff( [] ); 1416 1417 return [ 1418 [ 1419 function () { 1420 return "Saint Oliver Plunckett"; 1421 }, 1422 'Saint Oliver Plunckett' 1423 ], 1424 [ 'strlen', 'strlen' ], 1425 [ 'WANObjectCache::newEmpty', 'WANObjectCache::newEmpty' ], 1426 [ [ 'WANObjectCache', 'newEmpty' ], [ 'WANObjectCache', 'newEmpty' ] ], 1427 [ [ $hash, 'getLastError' ], [ $hash, 'getLastError' ] ], 1428 [ [ 1, 2, 3 ], [ 1, 2, 3 ] ] 1429 ]; 1430 } 1431 1432 /** 1433 * @covers WANObjectCache::getWithSetCallback() 1434 * @covers WANObjectCache::fetchOrRegenerate() 1435 * @dataProvider getBusyValues_Provider 1436 * @param mixed $busyValue 1437 * @param mixed $expected 1438 */ 1439 public function testBusyValueTypes( $busyValue, $expected ) { 1440 list( $cache, $bag ) = $this->newWanCache(); 1441 $key = wfRandomString(); 1442 1443 $mockWallClock = 1549343530.2053; 1444 $cache->setMockTime( $mockWallClock ); 1445 1446 $calls = 0; 1447 $func = function () use ( &$calls ) { 1448 ++$calls; 1449 return 418; 1450 }; 1451 1452 // Acquire a lock to verify that getWithSetCallback uses busyValue properly 1453 $this->setMutexKey( $bag, $key ); 1454 1455 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] ); 1456 $this->assertSame( $expected, $ret, 'busyValue used as expected' ); 1457 $this->assertSame( 0, $calls, 'busyValue was used' ); 1458 } 1459 1460 /** 1461 * @covers WANObjectCache::getMulti() 1462 */ 1463 public function testGetMulti() { 1464 list( $cache ) = $this->newWanCache(); 1465 1466 $value1 = [ 'this' => 'is', 'a' => 'test' ]; 1467 $value2 = [ 'this' => 'is', 'another' => 'test' ]; 1468 1469 $key1 = wfRandomString(); 1470 $key2 = wfRandomString(); 1471 $key3 = wfRandomString(); 1472 1473 $mockWallClock = 1549343530.2053; 1474 $priorTime = $mockWallClock; // reference time 1475 $cache->setMockTime( $mockWallClock ); 1476 1477 $cache->set( $key1, $value1, 5 ); 1478 $cache->set( $key2, $value2, 10 ); 1479 1480 $curTTLs = []; 1481 $this->assertSame( 1482 [ $key1 => $value1, $key2 => $value2 ], 1483 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs ), 1484 'Result array populated' 1485 ); 1486 1487 $this->assertCount( 2, $curTTLs, "Two current TTLs in array" ); 1488 $this->assertGreaterThan( 0, $curTTLs[$key1], "Key 1 has current TTL > 0" ); 1489 $this->assertGreaterThan( 0, $curTTLs[$key2], "Key 2 has current TTL > 0" ); 1490 1491 $cKey1 = wfRandomString(); 1492 $cKey2 = wfRandomString(); 1493 1494 $mockWallClock += 1; 1495 1496 $curTTLs = []; 1497 $this->assertSame( 1498 [ $key1 => $value1, $key2 => $value2 ], 1499 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ), 1500 "Result array populated even with new check keys" 1501 ); 1502 $t1 = $cache->getCheckKeyTime( $cKey1 ); 1503 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key 1 generated on miss' ); 1504 $t2 = $cache->getCheckKeyTime( $cKey2 ); 1505 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check key 2 generated on miss' ); 1506 $this->assertCount( 2, $curTTLs, "Current TTLs array set" ); 1507 $this->assertLessThanOrEqual( 0, $curTTLs[$key1], 'Key 1 has current TTL <= 0' ); 1508 $this->assertLessThanOrEqual( 0, $curTTLs[$key2], 'Key 2 has current TTL <= 0' ); 1509 1510 $mockWallClock += 1; 1511 1512 $curTTLs = []; 1513 $this->assertSame( 1514 [ $key1 => $value1, $key2 => $value2 ], 1515 $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ), 1516 "Result array still populated even with new check keys" 1517 ); 1518 $this->assertCount( 2, $curTTLs, "Current TTLs still array set" ); 1519 $this->assertLessThan( 0, $curTTLs[$key1], 'Key 1 has negative current TTL' ); 1520 $this->assertLessThan( 0, $curTTLs[$key2], 'Key 2 has negative current TTL' ); 1521 } 1522 1523 /** 1524 * @covers WANObjectCache::getMulti() 1525 * @covers WANObjectCache::processCheckKeys() 1526 * @param array $params 1527 * @dataProvider provideCoalesceAndMcrouterSettings 1528 */ 1529 public function testGetMultiCheckKeys( array $params ) { 1530 list( $cache ) = $this->newWanCache( $params ); 1531 1532 $checkAll = wfRandomString(); 1533 $check1 = wfRandomString(); 1534 $check2 = wfRandomString(); 1535 $check3 = wfRandomString(); 1536 $value1 = wfRandomString(); 1537 $value2 = wfRandomString(); 1538 1539 $mockWallClock = 1549343530.2053; 1540 $cache->setMockTime( $mockWallClock ); 1541 1542 // Fake initial check key to be set in the past. Otherwise we'd have to sleep for 1543 // several seconds during the test to assert the behaviour. 1544 foreach ( [ $checkAll, $check1, $check2 ] as $checkKey ) { 1545 $cache->touchCheckKey( $checkKey, WANObjectCache::HOLDOFF_TTL_NONE ); 1546 } 1547 1548 $mockWallClock += 0.100; 1549 1550 $cache->set( 'key1', $value1, 10 ); 1551 $cache->set( 'key2', $value2, 10 ); 1552 1553 $curTTLs = []; 1554 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1555 'key1' => $check1, 1556 $checkAll, 1557 'key2' => $check2, 1558 'key3' => $check3, 1559 ] ); 1560 $this->assertSame( 1561 [ 'key1' => $value1, 'key2' => $value2 ], 1562 $result, 1563 'Initial values' 1564 ); 1565 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key1'], 'Initial ttls' ); 1566 $this->assertLessThanOrEqual( 10.5, $curTTLs['key1'], 'Initial ttls' ); 1567 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key2'], 'Initial ttls' ); 1568 $this->assertLessThanOrEqual( 10.5, $curTTLs['key2'], 'Initial ttls' ); 1569 1570 $mockWallClock += 0.100; 1571 $cache->touchCheckKey( $check1 ); 1572 1573 $curTTLs = []; 1574 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1575 'key1' => $check1, 1576 $checkAll, 1577 'key2' => $check2, 1578 'key3' => $check3, 1579 ] ); 1580 $this->assertSame( 1581 [ 'key1' => $value1, 'key2' => $value2 ], 1582 $result, 1583 'key1 expired by check1, but value still provided' 1584 ); 1585 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 TTL expired' ); 1586 $this->assertGreaterThan( 0, $curTTLs['key2'], 'key2 still valid' ); 1587 1588 $cache->touchCheckKey( $checkAll ); 1589 1590 $curTTLs = []; 1591 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [ 1592 'key1' => $check1, 1593 $checkAll, 1594 'key2' => $check2, 1595 'key3' => $check3, 1596 ] ); 1597 $this->assertSame( 1598 [ 'key1' => $value1, 'key2' => $value2 ], 1599 $result, 1600 'All keys expired by checkAll, but value still provided' 1601 ); 1602 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 expired by checkAll' ); 1603 $this->assertLessThan( 0, $curTTLs['key2'], 'key2 expired by checkAll' ); 1604 } 1605 1606 /** 1607 * @covers WANObjectCache::get() 1608 * @covers WANObjectCache::processCheckKeys() 1609 */ 1610 public function testCheckKeyInitHoldoff() { 1611 list( $cache ) = $this->newWanCache(); 1612 1613 for ( $i = 0; $i < 500; ++$i ) { 1614 $key = wfRandomString(); 1615 $checkKey = wfRandomString(); 1616 // miss, set, hit 1617 $cache->get( $key, $curTTL, [ $checkKey ] ); 1618 $cache->set( $key, 'val', 10 ); 1619 $curTTL = null; 1620 $v = $cache->get( $key, $curTTL, [ $checkKey ] ); 1621 1622 $this->assertSame( 'val', $v ); 1623 $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (miss/set/hit)" ); 1624 } 1625 1626 for ( $i = 0; $i < 500; ++$i ) { 1627 $key = wfRandomString(); 1628 $checkKey = wfRandomString(); 1629 // set, hit 1630 $cache->set( $key, 'val', 10 ); 1631 $curTTL = null; 1632 $v = $cache->get( $key, $curTTL, [ $checkKey ] ); 1633 1634 $this->assertSame( 'val', $v ); 1635 $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (set/hit)" ); 1636 } 1637 } 1638 1639 /** 1640 * @covers WANObjectCache::get() 1641 * @covers WANObjectCache::processCheckKeys() 1642 */ 1643 public function testCheckKeyHoldoff() { 1644 list( $cache ) = $this->newWanCache(); 1645 $key = wfRandomString(); 1646 $checkKey = wfRandomString(); 1647 1648 $mockWallClock = 1549343530.2053; 1649 $cache->setMockTime( $mockWallClock ); 1650 $cache->touchCheckKey( $checkKey, 8 ); 1651 1652 $mockWallClock += 1; 1653 $cache->set( $key, 1, 60 ); 1654 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1655 $this->assertLessThan( 0, $curTTL, "Key in hold-off due to check key" ); 1656 1657 $mockWallClock += 3; 1658 $cache->set( $key, 1, 60 ); 1659 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1660 $this->assertLessThan( 0, $curTTL, "Key in hold-off due to check key" ); 1661 1662 $mockWallClock += 10; 1663 $cache->set( $key, 1, 60 ); 1664 $this->assertSame( 1, $cache->get( $key, $curTTL, [ $checkKey ] ) ); 1665 $this->assertGreaterThan( 0, $curTTL, "Key not in hold-off due to check key" ); 1666 } 1667 1668 /** 1669 * @covers WANObjectCache::delete 1670 * @covers WANObjectCache::relayDelete 1671 * @covers WANObjectCache::relayPurge 1672 */ 1673 public function testDelete() { 1674 list( $cache ) = $this->newWanCache(); 1675 $key = wfRandomString(); 1676 $value = wfRandomString(); 1677 $cache->set( $key, $value ); 1678 1679 $curTTL = null; 1680 $v = $cache->get( $key, $curTTL ); 1681 $this->assertSame( $value, $v, "Key was created with value" ); 1682 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); 1683 1684 $cache->delete( $key ); 1685 1686 $curTTL = null; 1687 $v = $cache->get( $key, $curTTL ); 1688 $this->assertSame( false, $v, "Deleted key has false value" ); 1689 $this->assertLessThan( 0, $curTTL, "Deleted key has current TTL < 0" ); 1690 1691 $cache->set( $key, $value . 'more' ); 1692 $v = $cache->get( $key, $curTTL ); 1693 $this->assertSame( false, $v, "Deleted key is tombstoned and has false value" ); 1694 $this->assertLessThan( 0, $curTTL, "Deleted key is tombstoned and has current TTL < 0" ); 1695 1696 $cache->set( $key, $value ); 1697 $cache->delete( $key, WANObjectCache::HOLDOFF_TTL_NONE ); 1698 1699 $curTTL = null; 1700 $v = $cache->get( $key, $curTTL ); 1701 $this->assertSame( false, $v, "Deleted key has false value" ); 1702 $this->assertSame( null, $curTTL, "Deleted key has null current TTL" ); 1703 1704 $cache->set( $key, $value ); 1705 $v = $cache->get( $key, $curTTL ); 1706 $this->assertSame( $value, $v, "Key was created with value" ); 1707 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" ); 1708 } 1709 1710 /** 1711 * @dataProvider getWithSetCallback_versions_provider 1712 * @covers WANObjectCache::getWithSetCallback() 1713 * @covers WANObjectCache::fetchOrRegenerate() 1714 * @param array $extOpts 1715 * @param bool $versioned 1716 */ 1717 public function testGetWithSetCallback_versions( array $extOpts, $versioned ) { 1718 list( $cache ) = $this->newWanCache(); 1719 1720 $key = wfRandomString(); 1721 $valueV1 = wfRandomString(); 1722 $valueV2 = [ wfRandomString() ]; 1723 1724 $wasSet = 0; 1725 $funcV1 = function () use ( &$wasSet, $valueV1 ) { 1726 ++$wasSet; 1727 1728 return $valueV1; 1729 }; 1730 1731 $priorValue = false; 1732 $priorAsOf = null; 1733 $funcV2 = function ( $oldValue, &$ttl, $setOpts, $oldAsOf ) 1734 use ( &$wasSet, $valueV2, &$priorValue, &$priorAsOf ) { 1735 $priorValue = $oldValue; 1736 $priorAsOf = $oldAsOf; 1737 ++$wasSet; 1738 1739 return $valueV2; // new array format 1740 }; 1741 1742 // Set the main key (version N if versioned) 1743 $wasSet = 0; 1744 $v = $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts ); 1745 $this->assertSame( $valueV1, $v, "Value returned" ); 1746 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1747 $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts ); 1748 $this->assertSame( 1, $wasSet, "Value not regenerated" ); 1749 $this->assertSame( $valueV1, $v, "Value not regenerated" ); 1750 1751 if ( $versioned ) { 1752 // Set the key for version N+1 format 1753 $verOpts = [ 'version' => $extOpts['version'] + 1 ]; 1754 } else { 1755 // Start versioning now with the unversioned key still there 1756 $verOpts = [ 'version' => 1 ]; 1757 } 1758 1759 // Value goes to secondary key since V1 already used $key 1760 $wasSet = 0; 1761 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1762 $this->assertSame( $valueV2, $v, "Value returned" ); 1763 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1764 $this->assertSame( false, $priorValue, "Old value not given due to old format" ); 1765 $this->assertSame( null, $priorAsOf, "Old value not given due to old format" ); 1766 1767 $wasSet = 0; 1768 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1769 $this->assertSame( $valueV2, $v, "Value not regenerated (secondary key)" ); 1770 $this->assertSame( 0, $wasSet, "Value not regenerated (secondary key)" ); 1771 1772 // Clear out the older or unversioned key 1773 $cache->delete( $key, 0 ); 1774 1775 // Set the key for next/first versioned format 1776 $wasSet = 0; 1777 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1778 $this->assertSame( $valueV2, $v, "Value returned" ); 1779 $this->assertSame( 1, $wasSet, "Value regenerated" ); 1780 1781 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts ); 1782 $this->assertSame( $valueV2, $v, "Value not regenerated (main key)" ); 1783 $this->assertSame( 1, $wasSet, "Value not regenerated (main key)" ); 1784 } 1785 1786 public static function getWithSetCallback_versions_provider() { 1787 return [ 1788 [ [], false ], 1789 [ [ 'version' => 1 ], true ] 1790 ]; 1791 } 1792 1793 /** 1794 * @covers WANObjectCache::useInterimHoldOffCaching 1795 * @covers WANObjectCache::getInterimValue 1796 * @dataProvider provideCoalesceAndMcrouterSettings 1797 * @param array $params 1798 */ 1799 public function testInterimHoldOffCaching( array $params ) { 1800 list( $cache, $bag ) = $this->newWanCache( $params ); 1801 1802 $mockWallClock = 1549343530.2053; 1803 $cache->setMockTime( $mockWallClock ); 1804 1805 $value = 'CRL-40-940'; 1806 $wasCalled = 0; 1807 $func = function () use ( &$wasCalled, $value ) { 1808 $wasCalled++; 1809 1810 return $value; 1811 }; 1812 1813 $cache->useInterimHoldOffCaching( true ); 1814 1815 $key = wfRandomString( 32 ); 1816 $cache->getWithSetCallback( $key, 60, $func ); 1817 $cache->getWithSetCallback( $key, 60, $func ); 1818 $this->assertSame( 1, $wasCalled, 'Value cached' ); 1819 1820 $cache->delete( $key ); // no value at all anymore and still locked 1821 1822 $mockWallClock += 0.001; // cached values will be newer than tombstone 1823 $cache->getWithSetCallback( $key, 60, $func ); 1824 $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim 1825 $cache->getWithSetCallback( $key, 60, $func ); 1826 $this->assertSame( 2, $wasCalled, 'Value interim cached' ); // reuses interim 1827 1828 $mockWallClock += 0.2; // interim key not brand new 1829 $cache->getWithSetCallback( $key, 60, $func ); 1830 $this->assertSame( 3, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim 1831 // Lock up the mutex so interim cache is used 1832 $this->setMutexKey( $bag, $key ); 1833 $cache->getWithSetCallback( $key, 60, $func ); 1834 $this->assertSame( 3, $wasCalled, 'Value interim cached (failed mutex)' ); 1835 $this->clearMutexKey( $bag, $key ); 1836 1837 $cache->useInterimHoldOffCaching( false ); 1838 1839 $wasCalled = 0; 1840 $key = wfRandomString( 32 ); 1841 $cache->getWithSetCallback( $key, 60, $func ); 1842 $cache->getWithSetCallback( $key, 60, $func ); 1843 $this->assertSame( 1, $wasCalled, 'Value cached' ); 1844 1845 $cache->delete( $key ); // no value at all anymore and still locked 1846 1847 $cache->getWithSetCallback( $key, 60, $func ); 1848 $this->assertSame( 2, $wasCalled, 'Value regenerated (got mutex)' ); 1849 $cache->getWithSetCallback( $key, 60, $func ); 1850 $this->assertSame( 3, $wasCalled, 'Value still regenerated (got mutex)' ); 1851 $cache->getWithSetCallback( $key, 60, $func ); 1852 $this->assertSame( 4, $wasCalled, 'Value still regenerated (got mutex)' ); 1853 // Lock up the mutex so interim cache is used 1854 $this->setMutexKey( $bag, $key ); 1855 $cache->getWithSetCallback( $key, 60, $func ); 1856 $this->assertSame( 5, $wasCalled, 'Value still regenerated (failed mutex)' ); 1857 } 1858 1859 /** 1860 * @covers WANObjectCache::touchCheckKey 1861 * @covers WANObjectCache::resetCheckKey 1862 * @covers WANObjectCache::getCheckKeyTime 1863 * @covers WANObjectCache::getMultiCheckKeyTime 1864 * @covers WANObjectCache::makePurgeValue 1865 * @covers WANObjectCache::parsePurgeValue 1866 */ 1867 public function testTouchKeys() { 1868 list( $cache ) = $this->newWanCache(); 1869 $key = wfRandomString(); 1870 1871 $mockWallClock = 1549343530.2053; 1872 $priorTime = $mockWallClock; // reference time 1873 $cache->setMockTime( $mockWallClock ); 1874 1875 $mockWallClock += 0.100; 1876 $t0 = $cache->getCheckKeyTime( $key ); 1877 $this->assertGreaterThanOrEqual( $priorTime, $t0, 'Check key auto-created' ); 1878 1879 $priorTime = $mockWallClock; 1880 $mockWallClock += 0.100; 1881 $cache->touchCheckKey( $key ); 1882 $t1 = $cache->getCheckKeyTime( $key ); 1883 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key created' ); 1884 1885 $t2 = $cache->getCheckKeyTime( $key ); 1886 $this->assertSame( $t1, $t2, 'Check key time did not change' ); 1887 1888 $mockWallClock += 0.100; 1889 $cache->touchCheckKey( $key ); 1890 $t3 = $cache->getCheckKeyTime( $key ); 1891 $this->assertGreaterThan( $t2, $t3, 'Check key time increased' ); 1892 1893 $t4 = $cache->getCheckKeyTime( $key ); 1894 $this->assertSame( $t3, $t4, 'Check key time did not change' ); 1895 1896 $mockWallClock += 0.100; 1897 $cache->resetCheckKey( $key ); 1898 $t5 = $cache->getCheckKeyTime( $key ); 1899 $this->assertGreaterThan( $t4, $t5, 'Check key time increased' ); 1900 1901 $t6 = $cache->getCheckKeyTime( $key ); 1902 $this->assertSame( $t5, $t6, 'Check key time did not change' ); 1903 } 1904 1905 /** 1906 * @covers WANObjectCache::getMulti() 1907 * @param array $params 1908 * @dataProvider provideCoalesceAndMcrouterSettings 1909 */ 1910 public function testGetWithSeveralCheckKeys( array $params ) { 1911 list( $cache, $bag ) = $this->newWanCache( $params ); 1912 $key = wfRandomString(); 1913 $tKey1 = wfRandomString(); 1914 $tKey2 = wfRandomString(); 1915 $value = 'meow'; 1916 1917 $mockWallClock = 1549343530.2053; 1918 $priorTime = $mockWallClock; // reference time 1919 $cache->setMockTime( $mockWallClock ); 1920 1921 // Two check keys are newer (given hold-off) than $key, another is older 1922 $this->setCheckKey( $bag, $tKey2, $priorTime - 3 ); 1923 $this->setCheckKey( $bag, $tKey2, $priorTime - 5 ); 1924 $this->setCheckKey( $bag, $tKey1, $priorTime - 30 ); 1925 $cache->set( $key, $value, 30 ); 1926 1927 $curTTL = null; 1928 $v = $cache->get( $key, $curTTL, [ $tKey1, $tKey2 ] ); 1929 $this->assertSame( $value, $v, "Value matches" ); 1930 $this->assertLessThan( -4.9, $curTTL, "Correct CTL" ); 1931 $this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" ); 1932 } 1933 1934 /** 1935 * @covers WANObjectCache::reap() 1936 * @covers WANObjectCache::reapCheckKey() 1937 */ 1938 public function testReap() { 1939 list( $cache, $bag ) = $this->newWanCache(); 1940 $vKey1 = wfRandomString(); 1941 $vKey2 = wfRandomString(); 1942 $tKey1 = wfRandomString(); 1943 $tKey2 = wfRandomString(); 1944 $value = 'moo'; 1945 1946 $knownPurge = time() - 60; 1947 $goodTime = microtime( true ) - 5; 1948 $badTime = microtime( true ) - 300; 1949 1950 $bag->set( 1951 'WANCache:v:' . $vKey1, 1952 [ 1953 0 => 1, 1954 1 => $value, 1955 2 => 3600, 1956 3 => $goodTime 1957 ] 1958 ); 1959 $bag->set( 1960 'WANCache:v:' . $vKey2, 1961 [ 1962 0 => 1, 1963 1 => $value, 1964 2 => 3600, 1965 3 => $badTime 1966 ] 1967 ); 1968 $bag->set( 1969 'WANCache:t:' . $tKey1, 1970 'PURGED:' . $goodTime 1971 ); 1972 $bag->set( 1973 'WANCache:t:' . $tKey2, 1974 'PURGED:' . $badTime 1975 ); 1976 1977 $this->assertSame( $value, $cache->get( $vKey1 ) ); 1978 $this->assertSame( $value, $cache->get( $vKey2 ) ); 1979 $cache->reap( $vKey1, $knownPurge, $bad1 ); 1980 $cache->reap( $vKey2, $knownPurge, $bad2 ); 1981 1982 $this->assertSame( false, $bad1 ); 1983 $this->assertTrue( $bad2 ); 1984 1985 $cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 ); 1986 $cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 ); 1987 $this->assertSame( false, $tBad1 ); 1988 $this->assertTrue( $tBad2 ); 1989 } 1990 1991 /** 1992 * @covers WANObjectCache::reap() 1993 */ 1994 public function testReap_fail() { 1995 $backend = $this->getMockBuilder( EmptyBagOStuff::class ) 1996 ->setMethods( [ 'get', 'changeTTL' ] )->getMock(); 1997 $backend->expects( $this->once() )->method( 'get' ) 1998 ->willReturn( [ 1999 0 => 1, 2000 1 => 'value', 2001 2 => 3600, 2002 3 => 300, 2003 ] ); 2004 $backend->expects( $this->once() )->method( 'changeTTL' ) 2005 ->willReturn( false ); 2006 2007 $wanCache = new WANObjectCache( [ 2008 'cache' => $backend 2009 ] ); 2010 2011 $isStale = null; 2012 $ret = $wanCache->reap( 'key', 360, $isStale ); 2013 $this->assertTrue( $isStale, 'value was stale' ); 2014 $this->assertSame( false, $ret, 'changeTTL failed' ); 2015 } 2016 2017 /** 2018 * @covers WANObjectCache::set() 2019 */ 2020 public function testSetWithLag() { 2021 list( $cache ) = $this->newWanCache(); 2022 $now = microtime( true ); 2023 $cache->setMockTime( $now ); 2024 2025 $v = 1; 2026 2027 $key = wfRandomString(); 2028 $opts = [ 'lag' => 300, 'since' => $now, 'walltime' => 0.1 ]; 2029 $cache->set( $key, $v, 30, $opts ); 2030 $this->assertSame( $v, $cache->get( $key ), "Repl-lagged value written." ); 2031 2032 $key = wfRandomString(); 2033 $opts = [ 'lag' => 300, 'since' => $now ]; 2034 $cache->set( $key, $v, 30, $opts ); 2035 $this->assertSame( $v, $cache->get( $key ), "Repl-lagged value written (no walltime)." ); 2036 2037 $key = wfRandomString(); 2038 $opts = [ 'lag' => 0, 'since' => $now - 300, 'walltime' => 0.1 ]; 2039 $cache->set( $key, $v, 30, $opts ); 2040 $this->assertSame( false, $cache->get( $key ), "Trx-lagged value written." ); 2041 2042 $key = wfRandomString(); 2043 $opts = [ 'lag' => 0, 'since' => $now - 300 ]; 2044 $cache->set( $key, $v, 30, $opts ); 2045 $this->assertSame( $v, $cache->get( $key ), "Trx-lagged value written (no walltime)." ); 2046 2047 $key = wfRandomString(); 2048 $opts = [ 'lag' => 5, 'since' => $now - 5, 'walltime' => 0.1 ]; 2049 $cache->set( $key, $v, 30, $opts ); 2050 $this->assertSame( false, $cache->get( $key ), "Trx-lagged value written." ); 2051 2052 $key = wfRandomString(); 2053 $opts = [ 'lag' => 5, 'since' => $now - 5 ]; 2054 $cache->set( $key, $v, 30, $opts ); 2055 $this->assertSame( false, $cache->get( $key ), "Lagged value not written (no walltime)." ); 2056 } 2057 2058 /** 2059 * @covers WANObjectCache::set() 2060 */ 2061 public function testWritePending() { 2062 list( $cache ) = $this->newWanCache(); 2063 $value = 1; 2064 2065 $key = wfRandomString(); 2066 $opts = [ 'pending' => true ]; 2067 $cache->set( $key, $value, 30, $opts ); 2068 $this->assertSame( false, $cache->get( $key ), "Pending value not written." ); 2069 } 2070 2071 public function testMcRouterSupport() { 2072 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2073 ->setMethods( [ 'set', 'delete' ] )->getMock(); 2074 $localBag->expects( $this->never() )->method( 'set' ); 2075 $localBag->expects( $this->never() )->method( 'delete' ); 2076 $wanCache = new WANObjectCache( [ 2077 'cache' => $localBag, 2078 'mcrouterAware' => true, 2079 'region' => 'pmtpa', 2080 'cluster' => 'mw-wan' 2081 ] ); 2082 $valFunc = function () { 2083 return 1; 2084 }; 2085 2086 // None of these should use broadcasting commands (e.g. SET, DELETE) 2087 $wanCache->get( 'x' ); 2088 $wanCache->get( 'x', $ctl, [ 'check1' ] ); 2089 $wanCache->getMulti( [ 'x', 'y' ] ); 2090 $wanCache->getMulti( [ 'x', 'y' ], $ctls, [ 'check2' ] ); 2091 $wanCache->getWithSetCallback( 'p', 30, $valFunc ); 2092 $wanCache->getCheckKeyTime( 'zzz' ); 2093 $wanCache->reap( 'x', time() - 300 ); 2094 $wanCache->reap( 'zzz', time() - 300 ); 2095 } 2096 2097 public function testMcRouterSupportBroadcastDelete() { 2098 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2099 ->setMethods( [ 'set' ] )->getMock(); 2100 $wanCache = new WANObjectCache( [ 2101 'cache' => $localBag, 2102 'mcrouterAware' => true, 2103 'region' => 'pmtpa', 2104 'cluster' => 'mw-wan' 2105 ] ); 2106 2107 $localBag->expects( $this->once() )->method( 'set' ) 2108 ->with( "/*/mw-wan/" . 'WANCache:v:' . "test" ); 2109 2110 $wanCache->delete( 'test' ); 2111 } 2112 2113 public function testMcRouterSupportBroadcastTouchCK() { 2114 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2115 ->setMethods( [ 'set' ] )->getMock(); 2116 $wanCache = new WANObjectCache( [ 2117 'cache' => $localBag, 2118 'mcrouterAware' => true, 2119 'region' => 'pmtpa', 2120 'cluster' => 'mw-wan' 2121 ] ); 2122 2123 $localBag->expects( $this->once() )->method( 'set' ) 2124 ->with( "/*/mw-wan/" . 'WANCache:t:' . "test" ); 2125 2126 $wanCache->touchCheckKey( 'test' ); 2127 } 2128 2129 public function testMcRouterSupportBroadcastResetCK() { 2130 $localBag = $this->getMockBuilder( EmptyBagOStuff::class ) 2131 ->setMethods( [ 'delete' ] )->getMock(); 2132 $wanCache = new WANObjectCache( [ 2133 'cache' => $localBag, 2134 'mcrouterAware' => true, 2135 'region' => 'pmtpa', 2136 'cluster' => 'mw-wan' 2137 ] ); 2138 2139 $localBag->expects( $this->once() )->method( 'delete' ) 2140 ->with( "/*/mw-wan/" . 'WANCache:t:' . "test" ); 2141 2142 $wanCache->resetCheckKey( 'test' ); 2143 } 2144 2145 public function testEpoch() { 2146 $bag = new HashBagOStuff(); 2147 $cache = new WANObjectCache( [ 'cache' => $bag ] ); 2148 $key = $cache->makeGlobalKey( 'The whole of the Law' ); 2149 2150 $now = microtime( true ); 2151 $cache->setMockTime( $now ); 2152 2153 $cache->set( $key, 'Do what thou Wilt' ); 2154 $cache->touchCheckKey( $key ); 2155 2156 $then = $now; 2157 $now += 30; 2158 $this->assertSame( 'Do what thou Wilt', $cache->get( $key ) ); 2159 $this->assertEqualsWithDelta( $then, $cache->getCheckKeyTime( $key ), 0.01, 'Check key init' ); 2160 2161 $cache = new WANObjectCache( [ 2162 'cache' => $bag, 2163 'epoch' => $now - 3600 2164 ] ); 2165 $cache->setMockTime( $now ); 2166 2167 $this->assertSame( 'Do what thou Wilt', $cache->get( $key ) ); 2168 $this->assertEqualsWithDelta( $then, $cache->getCheckKeyTime( $key ), 0.01, 'Check key kept' ); 2169 2170 $now += 30; 2171 $cache = new WANObjectCache( [ 2172 'cache' => $bag, 2173 'epoch' => $now + 3600 2174 ] ); 2175 $cache->setMockTime( $now ); 2176 2177 $this->assertSame( false, $cache->get( $key ), 'Key rejected due to epoch' ); 2178 $this->assertEqualsWithDelta( $now, $cache->getCheckKeyTime( $key ), 0.01, 'Check key reset' ); 2179 } 2180 2181 /** 2182 * @dataProvider provideAdaptiveTTL 2183 * @covers WANObjectCache::adaptiveTTL() 2184 * @param float|int $ago 2185 * @param int $maxTTL 2186 * @param int $minTTL 2187 * @param float $factor 2188 * @param int $adaptiveTTL 2189 */ 2190 public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) { 2191 list( $cache ) = $this->newWanCache(); 2192 $mtime = $ago ? time() - $ago : $ago; 2193 $margin = 5; 2194 $ttl = $cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor ); 2195 2196 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); 2197 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); 2198 2199 $ttl = $cache->adaptiveTTL( (string)$mtime, $maxTTL, $minTTL, $factor ); 2200 2201 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl ); 2202 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl ); 2203 } 2204 2205 public static function provideAdaptiveTTL() { 2206 return [ 2207 [ 3600, 900, 30, 0.2, 720 ], 2208 [ 3600, 500, 30, 0.2, 500 ], 2209 [ 3600, 86400, 800, 0.2, 800 ], 2210 [ false, 86400, 800, 0.2, 800 ], 2211 [ null, 86400, 800, 0.2, 800 ] 2212 ]; 2213 } 2214 2215 /** 2216 * @covers WANObjectCache::__construct 2217 * @covers WANObjectCache::newEmpty 2218 */ 2219 public function testNewEmpty() { 2220 $this->assertInstanceOf( 2221 WANObjectCache::class, 2222 WANObjectCache::newEmpty() 2223 ); 2224 } 2225 2226 /** 2227 * @covers WANObjectCache::setLogger 2228 */ 2229 public function testSetLogger() { 2230 list( $cache ) = $this->newWanCache(); 2231 $this->assertSame( null, $cache->setLogger( new Psr\Log\NullLogger ) ); 2232 } 2233 2234 /** 2235 * @covers WANObjectCache::getQoS 2236 */ 2237 public function testGetQoS() { 2238 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2239 ->setMethods( [ 'getQoS' ] )->getMock(); 2240 $backend->expects( $this->once() )->method( 'getQoS' ) 2241 ->willReturn( BagOStuff::QOS_UNKNOWN ); 2242 $wanCache = new WANObjectCache( [ 'cache' => $backend ] ); 2243 2244 $this->assertSame( 2245 $wanCache::QOS_UNKNOWN, 2246 $wanCache->getQoS( $wanCache::ATTR_EMULATION ) 2247 ); 2248 } 2249 2250 /** 2251 * @covers WANObjectCache::makeKey 2252 */ 2253 public function testMakeKey() { 2254 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2255 ->setMethods( [ 'makeKey' ] )->getMock(); 2256 $backend->expects( $this->once() )->method( 'makeKey' ) 2257 ->willReturn( 'special' ); 2258 2259 $wanCache = new WANObjectCache( [ 2260 'cache' => $backend 2261 ] ); 2262 2263 $this->assertSame( 'special', $wanCache->makeKey( 'a', 'b' ) ); 2264 } 2265 2266 /** 2267 * @covers WANObjectCache::makeGlobalKey 2268 */ 2269 public function testMakeGlobalKey() { 2270 $backend = $this->getMockBuilder( HashBagOStuff::class ) 2271 ->setMethods( [ 'makeGlobalKey' ] )->getMock(); 2272 $backend->expects( $this->once() )->method( 'makeGlobalKey' ) 2273 ->willReturn( 'special' ); 2274 2275 $wanCache = new WANObjectCache( [ 2276 'cache' => $backend 2277 ] ); 2278 2279 $this->assertSame( 'special', $wanCache->makeGlobalKey( 'a', 'b' ) ); 2280 } 2281 2282 public static function statsKeyProvider() { 2283 return [ 2284 [ 'domain:page:5', 'page' ], 2285 [ 'domain:main-key', 'main-key' ], 2286 [ 'domain:page:history', 'page' ], 2287 // Regression test for T232907 2288 [ 'domain:foo-bar-1.2:abc:v2', 'foo-bar-1_2' ], 2289 [ 'missingdomainkey', 'missingdomainkey' ] 2290 ]; 2291 } 2292 2293 /** 2294 * @dataProvider statsKeyProvider 2295 * @covers WANObjectCache::determineKeyClassForStats 2296 * @param string $key 2297 * @param string $class 2298 */ 2299 public function testStatsKeyClass( $key, $class ) { 2300 /** @var WANObjectCache $wanCache */ 2301 $wanCache = TestingAccessWrapper::newFromObject( new WANObjectCache( [ 2302 'cache' => new HashBagOStuff 2303 ] ) ); 2304 2305 $this->assertSame( $class, $wanCache->determineKeyClassForStats( $key ) ); 2306 } 2307 2308 /** 2309 * @covers WANObjectCache::makeMultiKeys 2310 */ 2311 public function testMakeMultiKeys() { 2312 list( $cache ) = $this->newWanCache(); 2313 2314 $ids = [ 1, 2, 3, 4, 4, 5, 6, 6, 7, 7 ]; 2315 $keyCallback = function ( $id, WANObjectCache $cache ) { 2316 return $cache->makeKey( 'key', $id ); 2317 }; 2318 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2319 2320 $expected = [ 2321 "local:key:1" => 1, 2322 "local:key:2" => 2, 2323 "local:key:3" => 3, 2324 "local:key:4" => 4, 2325 "local:key:5" => 5, 2326 "local:key:6" => 6, 2327 "local:key:7" => 7 2328 ]; 2329 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2330 2331 $ids = [ '1', '2', '3', '4', '4', '5', '6', '6', '7', '7' ]; 2332 $keyCallback = function ( $id, WANObjectCache $cache ) { 2333 return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' ); 2334 }; 2335 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2336 2337 $expected = [ 2338 "global:key:1:a:1:b" => '1', 2339 "global:key:2:a:2:b" => '2', 2340 "global:key:3:a:3:b" => '3', 2341 "global:key:4:a:4:b" => '4', 2342 "global:key:5:a:5:b" => '5', 2343 "global:key:6:a:6:b" => '6', 2344 "global:key:7:a:7:b" => '7' 2345 ]; 2346 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2347 } 2348 2349 /** 2350 * @covers WANObjectCache::makeMultiKeys 2351 */ 2352 public function testMakeMultiKeysIntString() { 2353 list( $cache ) = $this->newWanCache(); 2354 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7, '7' ]; 2355 $keyCallback = function ( $id, WANObjectCache $cache ) { 2356 return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' ); 2357 }; 2358 2359 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback ); 2360 2361 $expected = [ 2362 "global:key:1:a:1:b" => 1, 2363 "global:key:2:a:2:b" => 2, 2364 "global:key:3:a:3:b" => 3, 2365 "global:key:4:a:4:b" => 4, 2366 "global:key:5:a:5:b" => 5, 2367 "global:key:6:a:6:b" => 6, 2368 "global:key:7:a:7:b" => 7 2369 ]; 2370 $this->assertSame( $expected, iterator_to_array( $keyedIds ) ); 2371 } 2372 2373 /** 2374 * @covers WANObjectCache::makeMultiKeys 2375 */ 2376 public function testMakeMultiKeysCollision() { 2377 list( $cache ) = $this->newWanCache(); 2378 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7 ]; 2379 2380 $this->expectException( UnexpectedValueException::class ); 2381 $cache->makeMultiKeys( 2382 $ids, 2383 function ( $id ) { 2384 return "keymod:" . $id % 3; 2385 } 2386 ); 2387 } 2388 2389 /** 2390 * @covers WANObjectCache::multiRemap 2391 */ 2392 public function testMultiRemap() { 2393 list( $cache ) = $this->newWanCache(); 2394 $a = [ 'a', 'b', 'c' ]; 2395 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3 ]; 2396 2397 $this->assertSame( 2398 [ 'a' => 1, 'b' => 2, 'c' => 3 ], 2399 $cache->multiRemap( $a, $res ) 2400 ); 2401 2402 $a = [ 'a', 'b', 'c', 'c', 'd' ]; 2403 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3, 'keyD' => 4 ]; 2404 2405 $this->assertSame( 2406 [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ], 2407 $cache->multiRemap( $a, $res ) 2408 ); 2409 } 2410 2411 /** 2412 * @covers WANObjectCache::hash256 2413 */ 2414 public function testHash256() { 2415 list( $cache ) = $this->newWanCache( [ 'epoch' => 5 ] ); 2416 $this->assertEquals( 2417 'f402bce76bfa1136adc705d8d5719911ce1fe61f0ad82ddf79a15f3c4de6ec4c', 2418 $cache->hash256( 'x' ) 2419 ); 2420 2421 list( $cache ) = $this->newWanCache( [ 'epoch' => 50 ] ); 2422 $this->assertSame( 2423 'f79a126722f0a682c4c500509f1b61e836e56c4803f92edc89fc281da5caa54e', 2424 $cache->hash256( 'x' ) 2425 ); 2426 2427 list( $cache ) = $this->newWanCache( [ 'secret' => 'garden' ] ); 2428 $this->assertSame( 2429 '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', 2430 $cache->hash256( 'x' ) 2431 ); 2432 2433 list( $cache ) = $this->newWanCache( [ 'secret' => 'garden', 'epoch' => 3 ] ); 2434 $this->assertSame( 2435 '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093', 2436 $cache->hash256( 'x' ) 2437 ); 2438 } 2439 2440 /** 2441 * @covers WANObjectCache::getWithSetCallback() 2442 * @covers WANObjectCache::fetchOrRegenerate() 2443 * @covers WANObjectCache::get() 2444 * @covers WANObjectCache::set() 2445 * @dataProvider provideCoalesceAndMcrouterSettings 2446 * @param array $params 2447 * @param string|null $keyNeedle 2448 */ 2449 public function testCoalesceKeys( array $params, $keyNeedle ) { 2450 list( $cache, $bag ) = $this->newWanCache( $params ); 2451 $key = wfRandomString(); 2452 $callback = function () { 2453 return 2020; 2454 }; 2455 2456 $cache->getWithSetCallback( $key, 60, $callback ); 2457 $wrapper = TestingAccessWrapper::newFromObject( $bag ); 2458 foreach ( array_keys( $wrapper->bag ) as $bagKey ) { 2459 if ( $keyNeedle === null ) { 2460 $this->assertNotRegExp( '/[#{}]/', $bagKey, 'Respects "coalesceKeys"' ); 2461 } else { 2462 $this->assertStringContainsString( 2463 $keyNeedle, 2464 $bagKey, 2465 'Respects "coalesceKeys"' 2466 ); 2467 } 2468 } 2469 } 2470} 2471 2472class McrouterHashBagOStuff extends HashBagOStuff { 2473 public function set( $key, $value, $exptime = 0, $flags = 0 ) { 2474 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::set() calls 2475 // https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup 2476 if ( preg_match( '#^/\*/[^/]+/(.*)$#', $key, $m ) ) { 2477 $key = $m[1]; 2478 } 2479 2480 return parent::set( $key, $value, $exptime, $flags ); 2481 } 2482 2483 public function delete( $key, $flags = 0 ) { 2484 // Convert mcrouter broadcast keys to regular keys in HashBagOStuff::delete() calls 2485 // https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup 2486 if ( preg_match( '#^/\*/[^/]+/(.*)$#', $key, $m ) ) { 2487 $key = $m[1]; 2488 } 2489 2490 return parent::delete( $key, $flags ); 2491 } 2492} 2493 2494class NearExpiringWANObjectCache extends WANObjectCache { 2495 private const CLOCK_SKEW = 1; 2496 2497 protected function worthRefreshExpiring( $curTTL, $lowTTL ) { 2498 return ( $curTTL > 0 && ( $curTTL + self::CLOCK_SKEW ) < $lowTTL ); 2499 } 2500} 2501 2502class PopularityRefreshingWANObjectCache extends WANObjectCache { 2503 protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) { 2504 return ( ( $now - $asOf ) > $timeTillRefresh ); 2505 } 2506} 2507