memcache.inc 22.8 KB
Newer Older
1 2
<?php

3 4 5 6 7
/**
 * @file
 * Implementation of cache.inc with memcache logic included.
 */

8
require_once dirname(__FILE__) . '/dmemcache.inc';
9

10 11 12 13
/**
 * Defines the period after which wildcard clears are not considered valid.
 */
define('MEMCACHE_WILDCARD_INVALIDATE', 86400 * 28);
14 15
define('MEMCACHE_CONTENT_CLEAR', 'MEMCACHE_CONTENT_CLEAR');

16 17 18
/**
 * Implementation of cache.inc with memcache logic included
 */
19
class MemCacheDrupal implements DrupalCacheInterface {
20 21 22 23 24 25 26 27 28
  protected $memcache;

  /**
   * Constructs a MemCacheDrupal object.
   *
   * @param string $bin
   *   The cache bin for which the object is created.
   */
  public function __construct($bin) {
29 30
    $this->memcache = dmemcache_object($bin);
    $this->bin = $bin;
31

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
    // If page_cache_without_database is enabled, we have to manually load the
    // $conf array out of cache_bootstrap.
    static $variables_loaded = FALSE;
    if (!empty($GLOBALS['conf']['page_cache_without_database']) && !$variables_loaded) {
      global $conf;
      $variables_loaded = TRUE;
      // Try loading variables from cache. If that fails, we have to bootstrap
      // further in order to fetch them.
      if ($cached = cache_get('variables', 'cache_bootstrap')) {
        $variables = $cached->data;
        // Make sure variable overrides are applied, see variable_initialize().
        foreach ($conf as $name => $value) {
          $variables[$name] = $value;
        }
        $conf = $variables;
      }
      else {
        drupal_bootstrap(DRUPAL_BOOTSTRAP_VARIABLES, FALSE);
      }
    }

53
    $this->reloadVariables();
54
  }
55

56 57 58 59
  /**
   * Implements DrupalCacheInterface::get().
   */
  public function get($cid) {
60 61 62 63
    $cache = dmemcache_get($cid, $this->bin, $this->memcache);
    return $this->valid($cid, $cache) ? $cache : FALSE;
  }

64 65 66 67
  /**
   * Implements DrupalCacheInterface::getMultiple().
   */
  public function getMultiple(&$cids) {
68
    $results = dmemcache_get_multi($cids, $this->bin, $this->memcache);
69
    foreach ($results as $cid => $result) {
70
      if (!$this->valid($cid, $result)) {
71
        // This object has expired, so don't return it.
72 73 74
        unset($results[$cid]);
      }
    }
75 76 77
    // Remove items from the referenced $cids array that we are returning,
    // per the comment in cache_get_multiple() in includes/cache.inc.
    $cids = array_diff($cids, array_keys($results));
78 79 80
    return $results;
  }

81 82 83 84 85 86 87 88 89 90 91
  /**
   * Checks if a retrieved cache item is valid.
   *
   * @param string $cid
   *   The cache id of the item
   * @param mixed $cache
   *   The cache item.
   *
   * @return bool
   *   Whether the item is valid.
   */
92
  protected function valid($cid, $cache) {
93 94 95 96
    if ($cache) {
      $cache_tables = isset($_SESSION['cache_flush']) ? $_SESSION['cache_flush'] : NULL;
      // Items that have expired are invalid.
      if (isset($cache->expire) && $cache->expire !== CACHE_PERMANENT && $cache->expire <= $_SERVER['REQUEST_TIME']) {
97 98 99 100
        // If the memcache_stampede_protection variable is set, allow one
        // process to rebuild the cache entry while serving expired content to
        // the rest. Note that core happily returns expired cache items as valid
        // and relies on cron to expire them, but this is mostly reliant on its
101 102 103 104 105
        // use of CACHE_TEMPORARY which does not map well to memcache.
        // @see http://drupal.org/node/534092
        if (variable_get('memcache_stampede_protection', FALSE)) {
          // The process that acquires the lock will get a cache miss, all
          // others will get a cache hit.
106
          if ($this->lockInit() && $this->stampedeProtected($cid) && lock_acquire("memcache_$cid:$this->bin", variable_get('memcache_stampede_semaphore', 15))) {
107 108 109 110 111 112 113 114 115
            $cache = FALSE;
          }
        }
        else {
          $cache = FALSE;
        }
      }
      // Items created before the last full wildcard flush against this bin are
      // invalid.
116
      elseif (!isset($cache->created) || $cache->created <= $this->cache_flush) {
117 118 119 120 121 122 123 124 125 126 127 128 129
        $cache = FALSE;
      }
      // Items created before the last content flush on this bin i.e.
      // cache_clear_all() are invalid.
      elseif ($cache->expire != CACHE_PERMANENT && $cache->created + $this->cache_lifetime <= $this->cache_content_flush) {
        $cache = FALSE;
      }
      // Items cached before the cache was last flushed by the current user are
      // invalid.
      elseif ($cache->expire != CACHE_PERMANENT && is_array($cache_tables) && isset($cache_tables[$this->bin]) && $cache_tables[$this->bin] >= $cache->created) {
        // Cache item expired, return FALSE.
        $cache = FALSE;
      }
130 131 132 133 134 135
      // Temporary items created before the cache was last flushed by
      // cache_clear_all(NULL, $bin) are invalid.
      elseif (!empty($cache->temporary) && $cache->created + $this->cache_lifetime <= $this->cache_temporary_flush) {
        // CACHE_TEMPORARY item expired, return FALSE.
        $cache = FALSE;
      }
136 137
      // Finally, check for wildcard clears against this cid.
      else {
138
        if (!$this->wildcardValid($cid, $cache)) {
139 140
          $cache = FALSE;
        }
141 142 143
      }
    }

144 145 146
    // On cache misses, attempt to avoid stampedes when the
    // memcache_stampede_protection variable is enabled.
    if (!$cache) {
147
      if (variable_get('memcache_stampede_protection', FALSE) && $this->lockInit() && $this->stampedeProtected($cid) && !lock_acquire("memcache_$cid:$this->bin", variable_get('memcache_stampede_semaphore', 15))) {
148 149 150 151 152 153 154 155 156 157 158 159
        // Prevent any single request from waiting more than three times due to
        // stampede protection. By default this is a maximum total wait of 15
        // seconds. This accounts for two possibilities - a cache and lock miss
        // more than once for the same item. Or a cache and lock miss for
        // different items during the same request.
        // @todo: it would be better to base this on time waited rather than
        // number of waits, but the lock API does not currently provide this
        // information. Currently the limit will kick in for three waits of 25ms
        // or three waits of 5000ms.
        static $lock_count = 0;
        $lock_count++;
        if ($lock_count <= variable_get('memcache_stampede_wait_limit', 3)) {
160 161 162
          // The memcache_stampede_semaphore variable was used in previous
          // releases of memcache, but the max_wait variable was not, so by
          // default divide the semaphore value by 3 (5 seconds).
163
          lock_wait("memcache_$cid:$this->bin", variable_get('memcache_stampede_wait_time', 5));
164 165
          $cache = $this->get($cid);
        }
166 167
      }
    }
168 169

    return (bool) $cache;
170
  }
Steve Rude's avatar
Steve Rude committed
171

172 173 174 175
  /**
   * Implements DrupalCacheInterface::set().
   */
  public function set($cid, $data, $expire = CACHE_PERMANENT) {
176
    $created = round(microtime(TRUE), 3);
177

178
    // Create new cache object.
179
    $cache = new stdClass();
180 181 182
    $cache->cid = $cid;
    $cache->data = is_object($data) ? clone $data : $data;
    $cache->created = $created;
183
    // Record the previous number of wildcard flushes affecting our cid.
184
    $cache->flushes = $this->wildcardFlushes($cid);
185 186 187
    if ($expire == CACHE_TEMPORARY) {
      // Convert CACHE_TEMPORARY (-1) into something that will live in memcache
      // until the next flush.
188
      $cache->expire = REQUEST_TIME + 2591999;
189 190
      // This is a temporary cache item.
      $cache->temporary = TRUE;
191 192
    }
    // Expire time is in seconds if less than 30 days, otherwise is a timestamp.
193
    elseif ($expire != CACHE_PERMANENT && $expire < 2592000) {
194 195
      // Expire is expressed in seconds, convert to the proper future timestamp
      // as expected in dmemcache_get().
196
      $cache->expire = REQUEST_TIME + $expire;
197
      $cache->temporary = FALSE;
198 199 200
    }
    else {
      $cache->expire = $expire;
201
      $cache->temporary = FALSE;
202
    }
203

204 205 206 207 208 209
    // Manually track the expire time in $cache->expire.  When the object
    // expires, if stampede protection is enabled, it may be served while one
    // process rebuilds it. The ttl sent to memcache is set to the expire twice
    // as long into the future, this allows old items to be expired by memcache
    // rather than evicted along with a sufficient period for stampede
    // protection to continue to work.
210 211 212 213 214 215
    if ($cache->expire == CACHE_PERMANENT) {
      $memcache_expire = $cache->expire;
    }
    else {
      $memcache_expire = $cache->expire + (($cache->expire - REQUEST_TIME) * 2);
    }
216
    dmemcache_set($cid, $cache, $memcache_expire, $this->bin, $this->memcache);
217 218 219 220 221 222

    // Release lock if acquired in $this->valid().
    $lock = "memcache_$cid:$this->bin";
    if (variable_get('memcache_stampede_protection', FALSE) && isset($GLOBALS['locks'][$lock])) {
      lock_release("$lock");
    }
223
  }
224

225 226 227 228
  /**
   * Implements DrupalCacheInterface::clear().
   */
  public function clear($cid = NULL, $wildcard = FALSE) {
229 230 231 232
    if ($this->memcache === FALSE) {
      // No memcache connection.
      return;
    }
233 234 235 236 237

    // It is not possible to detect a cache_clear_all() call other than looking
    // at the backtrace unless http://drupal.org/node/81461 is added.
    $backtrace = debug_backtrace();
    if ($cid == MEMCACHE_CONTENT_CLEAR || (isset($backtrace[2]) && $backtrace[2]['function'] == 'cache_clear_all' && empty($backtrace[2]['args']))) {
238 239
      // Update the timestamp of the last global flushing of this bin.  When
      // retrieving data from this bin, we will compare the cache creation
240 241 242 243 244
      // time minus the cache_flush time to the cache_lifetime to determine
      // whether or not the cached item is still valid.
      $this->cache_content_flush = time();
      $this->variable_set('cache_content_flush_' . $this->bin, $this->cache_content_flush);
      if (variable_get('cache_lifetime', 0)) {
245 246 247
        // We store the time in the current user's session. We then simulate
        // that the cache was flushed for this user by not returning cached
        // data to this user that was cached before the timestamp.
248 249 250 251 252 253 254 255 256 257 258
        if (isset($_SESSION['cache_flush']) && is_array($_SESSION['cache_flush'])) {
          $cache_bins = $_SESSION['cache_flush'];
        }
        else {
          $cache_bins = array();
        }
        // Use time() rather than request time here for correctness.
        $cache_tables[$this->bin] = $this->cache_content_flush;
        $_SESSION['cache_flush'] = $cache_tables;
      }
    }
259
    if (empty($cid) || $wildcard === TRUE) {
260
      // system_cron() flushes all cache bins returned by hook_flush_caches()
261 262 263 264
      // with cache_clear_all(NULL, $bin); The expected behaviour in this case
      // is to perform garbage collection on expired cache items (which is
      // irrelevant to memcache) but also to remove all CACHE_TEMPORARY items.
      // @see https://api.drupal.org/api/drupal/includes!cache.inc/function/cache_clear_all/7
265
      if (!isset($cid)) {
266 267 268 269 270
        // Update the timestamp of the last CACHE_TEMPORARY clear. All
        // temporary cache items created before this will be invalidated.
        $this->cache_temporary_flush = time();
        $this->variable_set("cache_temporary_flush_$this->bin", $this->cache_temporary_flush);
        // Return early here as we do not want to register a wildcard flush.
271 272 273
        return;
      }
      elseif ($cid == '*') {
274 275
        $cid = '';
      }
276
      if (empty($cid)) {
277 278 279 280
        // Update the timestamp of the last global flushing of this bin.  When
        // retrieving data from this bin, we will compare the cache creation
        // time minus the cache_flush time to the cache_lifetime to determine
        // whether or not the cached item is still valid.
281 282 283
        $this->cache_flush = time();
        $this->variable_set("cache_flush_$this->bin", $this->cache_flush);
        $this->flushed = min($this->cache_flush, time() - $this->cache_lifetime);
284

285 286
        if ($this->cache_lifetime) {
          // We store the time in the current user's session which is saved into
287 288 289
          // the sessions table by sess_write().  We then simulate that the
          // cache was flushed for this user by not returning cached data to
          // this user that was cached before the timestamp.
290 291 292 293 294 295 296 297
          if (isset($_SESSION['cache_flush']) && is_array($_SESSION['cache_flush'])) {
            $cache_bins = $_SESSION['cache_flush'];
          }
          else {
            $cache_bins = array();
          }
          $cache_bins[$this->bin] = $this->cache_flush;
          $_SESSION['cache_flush'] = $cache_bins;
298 299 300
        }
      }
      else {
301
        // Register a wildcard flush for current cid.
302
        $this->wildcards($cid, TRUE);
303 304 305 306 307 308
      }
    }
    else {
      $cids = is_array($cid) ? $cid : array($cid);
      foreach ($cids as $cid) {
        dmemcache_delete($cid, $this->bin, $this->memcache);
309 310 311
      }
    }
  }
312

313
  /**
314 315 316 317 318 319 320 321 322 323
   * Sum of all matching wildcards.
   *
   * Checking any single cache item's flush value against this single-value sum
   * tells us whether or not a new wildcard flush has affected the cached item.
   *
   * @param string $cid
   *   The cache id to check.
   *
   * @return int
   *   Sum of all matching wildcards for the given cache id.
324
   */
325
  protected function wildcardFlushes($cid) {
326 327 328 329
    return array_sum($this->wildcards($cid));
  }

  /**
330 331
   * Retrieves all matching wildcards for the given cache id.
   *
332 333 334
   * Utilize multiget to retrieve all possible wildcard matches, storing
   * statically so multiple cache requests for the same item on the same page
   * load doesn't add overhead.
335
   */
336
  protected function wildcards($cid, $flush = FALSE) {
337
    static $wildcards = array();
338
    $matching = array();
339

340 341
    $length = strlen($cid);

342 343 344 345 346
    if (isset($this->wildcard_flushes[$this->bin]) && is_array($this->wildcard_flushes[$this->bin])) {
      // Wildcard flushes per table are keyed by a substring equal to the
      // shortest wildcard clear on the table so far. So if the shortest
      // wildcard was "links:foo:", and the cid we're checking for is
      // "links:bar:bar", then the key will be "links:bar:".
347 348
      $keys = array_keys($this->wildcard_flushes[$this->bin]);
      $wildcard_length = strlen(reset($keys));
349
      $wildcard_key = substr($cid, 0, $wildcard_length);
350

351 352 353 354 355 356 357
      // Determine which lookups we need to perform to determine whether or not
      // our cid was impacted by a wildcard flush.
      $lookup = array();

      // Find statically cached wildcards, and determine possibly matching
      // wildcards for this cid based on a history of the lengths of past
      // valid wildcard flushes in this bin.
358 359
      if (isset($this->wildcard_flushes[$this->bin][$wildcard_key])) {
        foreach ($this->wildcard_flushes[$this->bin][$wildcard_key] as $flush_length => $timestamp) {
360
          if ($length >= $flush_length && $timestamp >= ($_SERVER['REQUEST_TIME'] - $this->invalidate)) {
361
            $wildcard = '.wildcard-' . substr($cid, 0, $flush_length);
362 363
            if (isset($wildcards[$this->bin][$wildcard])) {
              $matching[$wildcard] = $wildcards[$this->bin][$wildcard];
364 365
            }
            else {
366
              $lookup[$wildcard] = $wildcard;
367
            }
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
          }
        }
      }

      // Do a multi-get to retrieve all possibly matching wildcard flushes.
      if (!empty($lookup)) {
        $values = dmemcache_get_multi($lookup, $this->bin, $this->memcache);
        if (is_array($values)) {
          // Prepare an array of matching wildcards.
          $matching = array_merge($matching, $values);
          // Store matches in the static cache.
          if (isset($wildcards[$this->bin])) {
            $wildcards[$this->bin] = array_merge($wildcards[$this->bin], $values);
          }
          else {
            $wildcards[$this->bin] = $values;
          }
          $lookup = array_diff_key($lookup, $values);
        }
387

388 389
        // Also store failed lookups in our static cache, so we don't have to
        // do repeat lookups on single page loads.
390 391
        foreach ($lookup as $key => $key) {
          $wildcards[$this->bin][$key] = 0;
392
        }
393 394
      }
    }
395

396
    if ($flush) {
397 398 399 400 401
      $key_length = $length;
      if (isset($this->wildcard_flushes[$this->bin])) {
        $keys = array_keys($this->wildcard_flushes[$this->bin]);
        $key_length = strlen(reset($keys));
      }
402
      $key = substr($cid, 0, $key_length);
403 404 405
      // Avoid too many calls to variable_set() by only recording a flush for
      // a fraction of the wildcard invalidation variable, per cid length.
      // Defaults to 28 / 4, or one week.
406
      if (!isset($this->wildcard_flushes[$this->bin][$key][$length]) || ($_SERVER['REQUEST_TIME'] - $this->wildcard_flushes[$this->bin][$key][$length] > $this->invalidate / 4)) {
407

408
        // If there are more than 50 different wildcard keys for this bin
409 410 411 412 413 414 415 416 417 418 419 420 421 422
        // shorten the key by one, this should reduce variability by
        // an order of magnitude and ensure we don't use too much memory.
        if (isset($this->wildcard_flushes[$this->bin]) && count($this->wildcard_flushes[$this->bin]) > 50) {
          $key = substr($cid, 0, $key_length - 1);
          $length = strlen($key);
        }

        // If this is the shortest key length so far, we need to remove all
        // other wildcards lengths recorded so far for this bin and start
        // again. This is equivalent to a full cache flush for this table, but
        // it ensures the minimum possible number of wildcards are requested
        // along with cache consistency.
        if ($length < $key_length) {
          $this->wildcard_flushes[$this->bin] = array();
423 424
          $this->variable_set("cache_flush_$this->bin", time());
          $this->cache_flush = time();
425 426 427 428
        }
        $key = substr($cid, 0, $key_length);
        $this->wildcard_flushes[$this->bin][$key][$length] = $_SERVER['REQUEST_TIME'];

429
        variable_set('memcache_wildcard_flushes', $this->wildcard_flushes);
430
      }
431
      $key = '.wildcard-' . $cid;
432
      if (isset($wildcards[$this->bin][$key])) {
433
        $wildcards[$this->bin][$key]++;
434 435
      }
      else {
436
        $wildcards[$this->bin][$key] = 1;
437
      }
438
      dmemcache_set($key, $wildcards[$this->bin][$key], 0, $this->bin);
439
    }
440
    return $matching;
441 442 443 444 445
  }

  /**
   * Check if a wildcard flush has invalidated the current cached copy.
   */
446
  protected function wildcardValid($cid, $cache) {
447 448
    // Previously cached content won't have ->flushes defined.  We could
    // force flush, but instead leave this up to the site admin.
449 450
    $flushes = isset($cache->flushes) ? (int) $cache->flushes : 0;
    if ($flushes < (int) $this->wildcardFlushes($cid)) {
451 452 453 454 455
      return FALSE;
    }
    return TRUE;
  }

456 457 458 459
  /**
   * Implements DrupalCacheInterface::isEmpty().
   */
  public function isEmpty() {
460 461
    // We do not know so err on the safe side?
    return FALSE;
462
  }
463

464 465
  /**
   * Helper function to load locking framework if not already loaded.
466 467 468 469
   *
   * @return bool
   *   Whether the locking system was initialized successfully. This must always
   *   return TRUE or throw an exception.
470
   */
471
  public function lockInit() {
472 473 474 475 476 477
    // On a cache miss when page_cache_without_database is enabled, we can end
    // up here without the lock system being initialized. Bootstrap drupal far
    // enough to load the lock system.
    if (!function_exists('lock_acquire')) {
      drupal_bootstrap(DRUPAL_BOOTSTRAP_VARIABLES, FALSE);
    }
478 479

    return TRUE;
480 481
  }

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
  /**
   * Determines whether stampede protection is enabled for a given bin/cid.
   *
   * Memcache stampede protection is primarily designed to benefit the following
   * caching pattern: a miss on a cache_get for a specific cid is immediately
   * followed by a cache_set for that cid. In cases where this pattern is not
   * followed, stampede protection can be disabled to avoid long hanging locks.
   * For example, a cache miss in Drupal core's module_implements() won't
   * execute a cache_set until drupal_page_footer() calls
   * module_implements_write_cache() which can occur much later in page
   * generation.
   *
   * @param string $cid
   *   The cache id of the data to retrieve.
   *
   * @return bool
   *   Returns TRUE if stampede protection is enabled for that particular cache
   *   bin/cid, otherwise FALSE.
   */
  protected function stampedeProtected($cid) {
    $ignore_settings = variable_get('memcache_stampede_protection_ignore', array(
      // Disable stampede protection for specific cids in 'cache_bootstrap'.
      'cache_bootstrap' => array(
        // The module_implements cache is written after finishing the request.
        'module_implements',
        // Variables have their own lock protection.
        'variables'
      ),
      // Disable stampede protection for cid prefix in 'cache'.
      'cache' => array(
        // I18n uses a class destructor to write the cache.
        'i18n:string:*',
      ),
    ));

    // Support ignoring an entire bin.
    if (in_array($this->bin, $ignore_settings)) {
      return FALSE;
    }

    // Support ignoring by cids.
    if (isset($ignore_settings[$this->bin])) {
      // Support ignoring specific cids.
      if (in_array($cid, $ignore_settings[$this->bin])) {
        return FALSE;
      }
      // Support ignoring cids starting with a suffix.
      foreach ($ignore_settings[$this->bin] as $ignore) {
        $split = explode('*', $ignore);
        if (count($split) > 1 && strpos($cid, $split[0]) === 0) {
          return FALSE;
        }
      }
    }

    return TRUE;
  }

540 541 542 543 544 545
  /**
   * Helper function to reload variables.
   *
   * This is used by the tests to verify that the cache object used the correct
   * settings.
   */
546
  public function reloadVariables() {
547 548 549 550 551
    $this->wildcard_flushes = variable_get('memcache_wildcard_flushes', array());
    $this->invalidate = variable_get('memcache_wildcard_invalidate', MEMCACHE_WILDCARD_INVALIDATE);
    $this->cache_lifetime = variable_get('cache_lifetime', 0);
    $this->cache_flush = variable_get('cache_flush_' . $this->bin);
    $this->cache_content_flush = variable_get('cache_content_flush_' . $this->bin, 0);
552
    $this->cache_temporary_flush = variable_get('cache_temporary_flush_' . $this->bin, 0);
553 554 555 556 557 558
    $this->flushed = min($this->cache_flush, REQUEST_TIME - $this->cache_lifetime);
  }

  /**
   * Re-implementation of variable_set() that writes through instead of clearing.
   */
559
  public function variable_set($name, $value) {
560 561
    global $conf;

562 563 564 565 566 567 568 569 570 571 572 573 574 575
    // When lots of writes happen in a short period of time db_merge can throw
    // errors. This should only happen if another request has written the variable
    // first, so we catch the error to prevent a fatal error.
    try {
      db_merge('variable')
        ->key(array('name' => $name))
        ->fields(array('value' => serialize($value)))
        ->execute();
    }
    catch (Exception $e) {
      // We can safely ignore the error, since it's likely a cache flush timestamp
      // which should still be accurate.
    }

576 577
    // If the variables are cached, get a fresh copy, update with the new value
    // and set it again.
578
    if ($cached = cache_get('variables', 'cache_bootstrap')) {
579 580
      $variables = $cached->data;
      $variables[$name] = $value;
581
      cache_set('variables', $variables, 'cache_bootstrap');
582 583 584 585
    }
    // If the variables aren't cached, there's no need to do anything.
    $conf[$name] = $value;
  }
586
}