Commit 85fa2a71 authored by Mike Carper's avatar Mike Carper Committed by Michael Carper
Browse files

Issue #3257162 by mikeytown2: https loopback failure

parent d055362d

README.txt

+9 −0
Changes for README.txt: 9 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -59,6 +59,15 @@ Settings page is located at: 
   and not queue up the request. Note that background callbacks will

   automatically be disabled if the site is in maintenance mode.



For use with a hostname that is using a self-signed certificate, to avoid the

certificate validation errors add the following to the site's settings.php file:



// httprl.

// Disable SSL cert validation.

$conf['drupal_ssl_context_options'] = array(

  'verify_peer' => FALSE,

);





API OVERVIEW

------------

examples/httprl.examples.php

+27 −27
Changes for examples/httprl.examples.php: 27 added lines, 27 removed lines.
Original line number Diff line number Diff line
@@ -12,10 +12,10 @@ Request http://drupal.org/. 
// Queue up the request.

httprl_request('http://drupal.org/');

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -25,11 +25,11 @@ Request http://drupal.org/robots.txt and save it to tmp folder. 
// Queue up the request.

httprl_request('http://drupal.org/robots.txt');

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();



// Save file if we got a 200 back.

if ($response['http://drupal.org/robots.txt']->code == 200) {

  file_put_contents('/tmp/robots.txt', $response['http://drupal.org/robots.txt']->data);

if ($request['http://drupal.org/robots.txt']->code == 200) {

  file_put_contents('/tmp/robots.txt', $request['http://drupal.org/robots.txt']->data);

}

?>
@@ -51,17 +51,17 @@ $url_node = httprl_build_url_self('node'); 
httprl_request($url_front, $options);

httprl_request($url_node, $options);

// Execute requests.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>





**Non Blocking HTTP Operations**



Request 10 URLs in a non blocking manner on this server. Checkout watchdog as

this should generate 10 404s and the $response object won't contain much info.

this should generate 10 404s and the $request object won't contain much info.



<?php

// Set the blocking mode.
@@ -80,10 +80,10 @@ for ($i=1; $i <= $max; $i++) { 
  httprl_request($url, $options);

}

// Execute requests.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -110,10 +110,10 @@ for ($i=1; $i <= $max; $i++) { 
// Queue up the requests.

httprl_request($urls, $options);

// Execute requests.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -143,10 +143,10 @@ for ($i=1; $i <= $max; $i++) { 
// Queue up the requests.

httprl_request($urls, $options);

// Execute requests.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -180,10 +180,10 @@ for ($i=1; $i <= $max; $i++) { 
// Queue up the requests.

httprl_request($urls, $options);

// Execute requests.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo out the results.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -214,7 +214,7 @@ $url_front = httprl_build_url_self(); 
// Queue up the request.

httprl_request($url_front, $options);

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo returned value from function callback.

echo $x;
@@ -247,7 +247,7 @@ $url_front = httprl_build_url_self(); 
// Queue up the request.

httprl_request($url_front, $options);

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo returned value from function callback.

echo $x;
@@ -285,7 +285,7 @@ $url_front = httprl_build_url_self(); 
// Queue up the request.

httprl_request($url_front, $options);

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();



// Echo what was returned and printed from function callback.

echo $x . "<br />\n";
@@ -318,10 +318,10 @@ $options = array( 
httprl_request($urls, $options);



// Execute requests.

$response = httprl_send_request();

$requests = httprl_send_request();



// Print what was done.

echo httprl_pr($response);

echo httprl_pr($requests);



function need_two_good_results($id, &$responses) {

  static $counter = 0;
@@ -380,9 +380,9 @@ $url_front = httprl_build_url_self(); 
// Queue up the request.

httprl_request($url_front, $options);

// Execute request.

$response = httprl_send_request();

$request = httprl_send_request();

// Echo what was returned.

echo httprl_pr($response);

echo httprl_pr($request);

?>
@@ -425,17 +425,17 @@ $options = array( 
httprl_request($urls, $options);

// Execute request.

echo round(timer_read('page')/1000, 3) . " - Time taken to get requests ready.<br> \n";

$response = httprl_send_request();

echo strtoupper(var_export($response, TRUE)) . " - Output from first httprl_send_request() call<br> \n";

$request = httprl_send_request();

echo strtoupper(var_export($request, TRUE)) . " - Output from first httprl_send_request() call<br> \n";



echo round(timer_read('page')/1000, 3) . " - Time taken to send out all fwrites().<br> \n";

sleep(2);

echo round(timer_read('page')/1000, 3) . " - Time taken for sleep(2).<br> \n";



$response = httprl_send_request();

$request = httprl_send_request();

echo round(timer_read('page')/1000, 3) . " - Time taken for all freads().<br> \n";

echo "Output from second httprl_send_request() below:<br> \n<br> \n";

echo httprl_pr($response);

echo httprl_pr($request);

?>

httprl.async.inc

+5 −0
Changes for httprl.async.inc: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -88,6 +88,11 @@ function httprl_async_page() { 
    }

  }



  // If on 64bit system give php 3 GB of ram.

  if (PHP_INT_SIZE > 4) {

    ini_set('memory_limit', httprl_variable_get('httprl_async_mem_limit', HTTPRL_ASYNC_MEM_LIMIT));

  }



  // Run the function.

  if (!empty($_POST['function'])) {

    $data = httprl_run_function($_POST['function'], $args);

httprl.module

+148 −33
Changes for httprl.module: 148 added lines, 33 removed lines.
Original line number Diff line number Diff line
@@ -147,6 +147,16 @@ define('HTTPRL_NON_BLOCKING_FCLOSE_DELAY', 0); 
 */

define('HTTPRL_SERVER_PORT', '');



/**

 * Just for loopback 127.0.0.1; ignore SSL errors allowing the connection.

 */

define('HTTPRL_LOOPBACK_IGNORE_SSL_ERRORS', TRUE);



/**

 * If on 64bit system give php 3 GB of ram for async callbacks.

 */

define('HTTPRL_ASYNC_MEM_LIMIT', '3G');



/**

 * Implements hook_url_inbound_alter().

 */
@@ -259,7 +269,7 @@ function httprl_override_core($url, $options = array()) { 
  // Execute request.

  $response = httprl_send_request();

  // Send back results.

  return is_array($response) && is_string($url) && array_key_exists($url, $response) ? $response[$url] : is_array($response) ? array_pop($response) : $response;

  return (is_array($response) && is_string($url) && array_key_exists($url, $response)) ? $response[$url] : (is_array($response) ? array_pop($response) : $response);

}



/**
@@ -537,6 +547,7 @@ function httprl_set_default_options(array &$options) { 
    'chunk_size_write' => 1024,

    'async_connect' => TRUE,

    'ping_db' => 20,

    'ignore_empty_chunk' => FALSE,

    'url_encoding' => array(),

  );
@@ -556,6 +567,12 @@ function httprl_set_default_options(array &$options) { 
    'User-Agent' => 'Drupal (+http://drupal.org/)',

    'Connection' => 'close',

  );

  if (isset($options['headers']['User-Agent']) && $options['headers']['User-Agent'] === FALSE) {

    unset($options['headers']['User-Agent']);

  }

  if (isset($options['headers']['Connection']) && $options['headers']['Connection'] === FALSE) {

    unset($options['headers']['Connection']);

  }



  // Defaults for url encoding the url.

  // Use %20 for spaces in the path and in the query string.
@@ -571,6 +588,9 @@ function httprl_set_default_options(array &$options) { 
  if (!isset($options['headers']['Referer']) && !empty($options['referrer'])) {

    if (function_exists('request_uri')) {

      $options['headers']['Referer'] = $base_root . request_uri();

      if (strpos($options['headers']['Referer'], 'httprl_async_function_callback') === TRUE) {

        $options['headers']['Referer'] = $_SERVER['HTTP_REFERER'];

      }

    }

  }
@@ -775,19 +795,11 @@ function httprl_handle_data(&$options) { 
      $options['data'] = $data_stream;

    }

    // No files passed in, url-encode the data.

    elseif (empty($options['data']['files']) || !is_array($options['data']['files'])) {

      $options['data'] = http_build_query($options['data'], '', '&');



      // Set the Content-Type to application/x-www-form-urlencoded if the data

      // is not empty, the Content-Type is not set, and the method is POST or

      // PUT.

      if (!empty($options['data']) && !isset($options['headers']['Content-Type']) && ($options['method'] == 'POST' || $options['method'] == 'PUT')) {

        $options['headers']['Content-Type'] = 'application/x-www-form-urlencoded';

      }

    }

    else {

    elseif (!empty($options['data']['file']) || !empty($options['data']['files'])) {

      $data_stream = '';

      $data_array = $options['data'];

      // Add files to the request.

      if (isset($options['data']['files'])) {

        foreach ($options['data']['files'] as $field_name => $info) {

          $multi_field = '[]';

          // Convert $info into an array if it's a string.
@@ -816,8 +828,26 @@ function httprl_handle_data(&$options) { 
          }

        }

        // Remove files from the data array as they have already been added.

      $data_array = $options['data'];

        unset($data_array['files']);

      }

      elseif (isset($options['data']['file'])) {

        $fullpath = $options['data']['file'];

        // Strip '@' from the start of the path (cURL requirement).

        if (substr($fullpath, 0, 1) == "@") {

          $fullpath = substr($fullpath, 1);

        }

        $filename = basename($fullpath);

        // TODO: mime detection.

        $mimetype = 'application/octet-stream';



        // Build the data-stream for this file.

        $data_stream .= '--' . HTTPRL_MULTIPART_BOUNDARY . "\r\n";

        $data_stream .= 'Content-Disposition: form-data; name="file"; filename="' . $filename . "\"\r\n";

        $data_stream .= 'Content-Transfer-Encoding: binary' . "\r\n";

        $data_stream .= 'Content-Type: ' . $mimetype . "\r\n\r\n";

        $data_stream .= file_get_contents($fullpath) . "\r\n";

        unset($data_array['file']);

      }

      // Add fields to the request too: $_POST['foo'] = 'bar'.

      httprl_multipart_encoder($data_stream, $data_array);
@@ -831,6 +861,17 @@ function httprl_handle_data(&$options) { 
        $options['headers']['Content-Type'] = 'multipart/form-data; boundary=' . HTTPRL_MULTIPART_BOUNDARY;

      }

    }

    // No files passed in, url-encode the data.

    else {

      $options['data'] = http_build_query($options['data'], '', '&');



      // Set the Content-Type to application/x-www-form-urlencoded if the data

      // is not empty, the Content-Type is not set, and the method is POST or

      // PUT.

      if (!empty($options['data']) && !isset($options['headers']['Content-Type']) && ($options['method'] == 'POST' || $options['method'] == 'PUT')) {

        $options['headers']['Content-Type'] = 'application/x-www-form-urlencoded';

      }

    }

  }



  // Only add Content-Length if we actually have any content or if it is a POST
@@ -943,7 +984,7 @@ function httprl_stream_connection_error_formatter($errno, $errstr, &$result) { 
      require_once DRUPAL_ROOT . '/includes/unicode.inc';

    }

    else {

      require_once './includes/unicode.inc';

      require_once $_SERVER['DOCUMENT_ROOT'] . '/includes/unicode.inc';

    }



    // Convert error message to utf-8. Using ISO-8859-1 (Latin-1) as source
@@ -1013,6 +1054,14 @@ function httprl_establish_stream_connection(&$result) { 
      $timeout += $result->options['connect_timeout'];

    }



    // Check for timeout here in the connection loop.

    $now_time = microtime(TRUE);

    $running_time = $now_time - $start_time;

    if ($running_time > $timeout) {

      break;

    }

    ini_set('default_socket_timeout', $timeout);



    // Open the connection.

    if (empty($result->options['context'])) {

      $result->fp = @stream_socket_client($result->socket, $errno, $errstr, $timeout, $result->flags);
@@ -1143,6 +1192,10 @@ function httprl_establish_stream_connection(&$result) { 
 *   - ping_db: After X amount of time, ping the DB with a simple query in order

 *     to keep the connection alive. Default is every 20 seconds. Set to 0 to

 *     disable.

 *   - ignore_empty_chunk: an empty chunk returned from a stream will:

 *     default FALSE - make the connection not 'alive' and stop reading;

 *     TRUE - not stop a stream from being 'alive' and will continue to read

 *     until EOF or timeout.

 *

 * @return array

 *   Array where key is the URL and the value is the return value from
@@ -1299,7 +1352,7 @@ function httprl_send_request($results = NULL) { 
      continue;

    }



    if (!empty($result->cached)) {

    if (!empty($result->cached->url)) {

      // Used the cached data.

      $output[$result->cached->url] = $result->cached;

      unset($responses[$id]);
@@ -1632,8 +1685,13 @@ function httprl_send_request($results = NULL) { 


          // Get stream data.

          $info = stream_get_meta_data($r);

          $alive = !$info['eof'] && !feof($r) && !$info['timed_out'] && strlen($chunk);

          if (!$alive && $responses[$id]->status !== 'Request sent, waiting for response.') {

          $alive = !$info['eof'] && !feof($r) && !$info['timed_out'];

          // Determine if an empty chunk of data means the connection should

          // stop reading even though EOF or timeout has not occurred.

          if ($alive && !$responses[$id]->options['ignore_empty_chunk'] && !strlen($chunk)) {

            $alive = FALSE;

          }

          if (!$alive) {

            if ($responses[$id]->status == 'Connecting.') {

              $responses[$id]->error = $t('Connection refused by destination. TCP.');

              $responses[$id]->code = HTTPRL_CONNECTION_REFUSED;
@@ -1902,8 +1960,8 @@ function httprl_parse_data(&$result) { 
        $original_location = @parse_url($result->url);

        // Assume request is to self if none of this was setup correctly.

        $location['scheme'] = !empty($location['scheme']) ? $location['scheme'] : $original_location['scheme'];

        $location['host'] = !empty($location['host']) ? $location['host'] : (!empty($original_location['host']) ? $original_location['host'] : $hostname);

        $location['port'] = !empty($location['port']) ? $location['port'] : (!empty($original_location['port']) ? $original_location['port'] : '');

        $location['host'] = !empty($location['host']) ? $location['host'] : ((!empty($original_location['host']) ? $original_location['host'] : $hostname));

        $location['port'] = !empty($location['port']) ? $location['port'] : ((!empty($original_location['port']) ? $original_location['port'] : ''));

        $location = httprl_glue_url($location);

      }

      else {
@@ -1923,8 +1981,30 @@ function httprl_parse_data(&$result) { 
        $result->error = $t('Maximum allowed redirects exhausted.');

      }

      else {

        // Prepare the received cookie values from the previous request and pass

        // them on to the next.

        $cookie_string = '';

        $parsed_location = parse_url($location);

        if(isset($result->cookies)) {

          foreach ($result->cookies as $cookie) {

            // Only add the cookie values if the cookie domain matches.

            $cookie_domain = ltrim($cookie['domain'], '.');

            if (strpos($parsed_location['host'], $cookie_domain) !== FALSE) {

              if (!empty($cookie_string)) {

                $cookie_string .= ';';

              }

              $cookie_string .= $cookie['name'] . '=' . $cookie['value'];

            }

          }



          // If the cookie string is changed and thus not empty we want to set

          // it in the options array.

          if (!empty($cookie_string)) {

            $result->options['headers']['Cookie'] = $cookie_string;

          }

        }



        // Redirect to the new location.

        // TODO: Send cookies in the redirect request if domain/path match.

        $result->options['max_redirects']--;

        if (isset($result->options['headers']['Referer'])) {

          $result->options['headers']['Referer'] = $result->url;
@@ -2022,6 +2102,7 @@ function httprl_post_processing($id, &$responses, &$output, $time_left = NULL) { 
      }

    }

    @fclose($result->fp);

    unset($result->fp);

  }



  // Set timeout.
@@ -2303,7 +2384,18 @@ function httprl_queue_background_callback(&$args, &$result = NULL) { 
    ),

    'blocking' => $mode,

    'method' => 'POST',

    'referrer' => 'TRUE',

  );



  if (empty($callback_options['context']) && httprl_variable_get('httprl_loopback_ignore_ssl_errors', HTTPRL_LOOPBACK_IGNORE_SSL_ERRORS)) {

    // Set context for loopback 127.0.0.1 to ignore any SSL issues.

    $drupal_ssl_context_options = array(

      'verify_peer' => FALSE,

      'verify_peer_name' => FALSE,

      'allow_self_signed' => TRUE,

    );

    $options['context'] = stream_context_create(['ssl' => $drupal_ssl_context_options]);

  }

  if (isset($callback_options['printed'])) {

    $options['internal_states']['background_function_printed'] = &$callback_options['printed'];

  }
@@ -2766,26 +2858,29 @@ function httprl_lock_release($name) { 
 * @return mixed

 *   $data

 */

function httprl_print_empty(&$data, $level = 0) {

function httprl_print_empty(&$data, $level = 0, $output_plain_text) {

  $level++;

  if ($level < 5) {

    if (is_object($data)) {

      $new_object = new stdClass();

      $new_object->__original_class_name__ = get_class($data);

      foreach ($data as $key => $values) {

        $new_object->{$key} = httprl_print_empty($values, $level);

        $new_object->{$key} = httprl_print_empty($values, $level, $output_plain_text);

      }

      $data = $new_object;

    }

    elseif (is_array($data)) {

      foreach ($data as &$values) {

        $values = httprl_print_empty($values, $level);

        $values = httprl_print_empty($values, $level, $output_plain_text);

      }

    }

    elseif (is_bool($data) || strlen((string) $data) == 0) {

      $data = strtoupper(var_export($data, TRUE));

    }

    elseif (is_string($data) && strlen($data) > HTTPRL_PR_MAX_STRING_LENGTH) {

    elseif (!$output_plain_text

          && is_string($data)

          && strlen($data) > HTTPRL_PR_MAX_STRING_LENGTH

    ) {

      // Do not try to print a string longer than 256KB.

      // Some browsers have issues with very long documents.

      $data = substr($data, 0, HTTPRL_PR_MAX_STRING_LENGTH);
@@ -2815,7 +2910,7 @@ function httprl_pr($input) { 
  $input = func_get_args();



  // If bool or strlen = 0 use var_export on that variable.

  $data = httprl_print_empty($input);

  $data = httprl_print_empty($input, 0, $output_plain_text);



  // Merge into base array if only one argument passed in.

  if (count($data) == 1) {
@@ -3596,8 +3691,8 @@ function httprl_call_user_func_array_cache($callback, array $param_arr = array() 
  else {

    $cid = __FUNCTION__ . ':' . drupal_hash_base64(serialize(array($callback, $param_arr)));

  }



  $cache = cache_get($cid, $options['bin']);



  // Don't use the cache if it's not there or the cached item is older than

  // the max cache lifetime.

  if (empty($options['return_null_cache_miss']) && (!isset($cache->data) || $cache->created < REQUEST_TIME - $options['cache_lifetime_max'])) {
@@ -3633,13 +3728,24 @@ function httprl_call_user_func_array_cache($callback, array $param_arr = array() 
      && lock_may_be_available($cid . ':run_function')

      && httprl_acquire_headless_lock($cid, $options['lock_timeout'])

    ) {

      // Set max to zero so this gets regenerated in the background.

      // Also do not skip on a cache miss.

      $options['cache_lifetime_max'] = 0;

      $options['return_null_cache_miss'] = FALSE;

      // Get all function arguments.

      $args = array($callback, $param_arr, $options);

      // Call this function again in another process with max lifetime set to

      // zero.



      if (!httprl_is_background_callback_capable()) {

        // Call this function again in a shutdown function.

        _httprl_run_functions_once_on_shutdown_array_cache($args);

      }

      else {

        // Call this function again in another process.

        httprl_call_user_func_array_async('httprl_call_user_func_array_cache', $args);

      }

    }

  }



  return isset($cache->data) ? $cache->data : NULL;

}
@@ -3662,8 +3768,9 @@ function _httprl_run_functions_once_on_shutdown_array_cache($args = array()) { 
    unset($args);

    $will_run = array();

    $already_ran = array();

    foreach ($run_list as $args) {

    foreach ($run_list as $key => $args) {

      if (empty($args)) {

        unset($run_list[$key]);

        continue;

      }

      if (is_string($args[0])) {
@@ -3678,8 +3785,10 @@ function _httprl_run_functions_once_on_shutdown_array_cache($args = array()) { 
      }

    }

    if (!empty($will_run)) {

      // Last chance for async call; odds are this will be a sync call if here.

      httprl_call_user_func_array_async('_httprl_run_cache_functions', array($will_run));

    }

    unset($run_list[$key]);

  }

}
@@ -3690,11 +3799,17 @@ function _httprl_run_functions_once_on_shutdown_array_cache($args = array()) { 
 *   The callable to be called.

 */

function _httprl_run_cache_functions($functions_to_cache) {

  // Cut connection to browser.

  @ob_end_flush();

  @ob_flush();

  @flush();

  sleep(1);



  if (function_exists('fastcgi_finish_request')) {

    fastcgi_finish_request();

  }



  foreach ($functions_to_cache as $args) {

    // Set max to zero so this gets regenerated in the background.

    // Also do not skip on a cache miss.

    $args[2]['cache_lifetime_max'] = 0;

    $args[2]['return_null_cache_miss'] = FALSE;

    call_user_func_array('httprl_call_user_func_array_cache', $args);

  }

}