- Many excellent news aggregator improvements by Kjartan:
    + Added drupal_http_request().
    + Replaced rssfeeds with OPML feed subscription list.
    + Added support for pubDate.
    + Added support for conditional gets using ETag and Last-Modified.

CHANGELOG

@@ -4,6 +4,9 @@ Drupal x.x.x, xxxx-xx-xx
 - added support for the MetaWeblog API and MovableType extensions.
 - added a file API: enables better document management.
 - improved the watchdog and search module to log search keys.
+- news aggregator:
+    * added support for conditional GET.
+    * added OPML feed subscription list.
 - comment module:
     * made it possible to disable the "comment viewing controls".
 - performance:

database/database.mssql

@@ -104,10 +104,12 @@ CREATE TABLE [dbo].[feed] (
   [title] [varchar] (255) NOT NULL ,
   [url] [varchar] (255) NOT NULL ,
   [refresh] [int] NOT NULL ,
-  [timestamp] [int] NULL ,
+  [checked] [int] MOT NULL ,
   [attributes] [varchar] (255) NULL ,
   [link] [varchar] (255) NULL ,
-  [description] [varchar] (8000) NULL
+  [description] [varchar] (8000) NULL ,
+  [etag] [varchar] (255) NULL ,
+  [modified] [int] NOT NULL
 ) ON [PRIMARY]
 GO
 

database/database.mysql

@@ -156,10 +156,12 @@ CREATE TABLE feed (
   title varchar(255) NOT NULL default '',
   url varchar(255) NOT NULL default '',
   refresh int(10) NOT NULL default '0',
-  timestamp int(10) NOT NULL default '0',
+  checked int(10) NOT NULL default '0',
   attributes varchar(255) NOT NULL default '',
   link varchar(255) NOT NULL default '',
   description longtext NOT NULL,
+  etag varchar(255) NOT NULL default '',
+  modified int(10) NOT NULL default '0',
   PRIMARY KEY  (fid),
   UNIQUE KEY link (url),
   UNIQUE KEY title (title)

database/database.pgsql

@@ -154,10 +154,12 @@ CREATE TABLE feed (
   title varchar(255) NOT NULL default '',
   url varchar(255) NOT NULL default '',
   refresh integer NOT NULL default '0',
-  timestamp integer NOT NULL default '0',
+  checked integer NOT NULL default '0',
   attributes varchar(255) NOT NULL default '',
   link varchar(255) NOT NULL default '',
   description text NOT NULL default '',
+  etag varchar(255) NOT NULL default '',
+  modified integer NOT NULL default '0',
   PRIMARY KEY  (fid),
   UNIQUE (title),
   UNIQUE (url)

includes/common.inc

@@ -187,6 +187,127 @@ function drupal_not_found() {
     print theme("page", '<h1>'. t('Page not found') .'</h1>');
   }
 }
+
+/**
+ * Flexible and powerful HTTP client implementation. Allows to GET, POST, PUT
+ * or any other HTTP requests. Handles redirects.
+ *
+ * @param $url A string containing a fully qualified URI.
+ * @param $headers An array containing a HTTP header => value pair.
+ * @param $method A string defining the HTTP request to use.
+ * @param $data A string containing data to include in the request.
+ * @param $retry An integer representing how many times to retry the request
+ *   in case of a redirect.
+ * @return An object containing the HTTP request headers, response code,
+ *   headers, data, and redirect status.
+ */
+function drupal_http_request($url, $headers = array(), $method = 'GET', $data = NULL, $retry = 3) {
+  // Parse the URL, and make sure we can handle the schema
+  $uri = parse_url($url);
+  switch ($uri['scheme']) {
+    case 'http':
+      $fp = @fsockopen($uri['host'], ($uri['port'] ? $uri['port'] : 80), $errno, $errstr, 15);
+      break;
+    case 'https':
+      // Note: only works for PHP 4.3 compiled with openssl
+      $fp = @fsockopen("ssl://$uri[host]", ($uri['port'] ? $uri['port'] : 443), $errno, $errstr, 20);
+      break;
+    default:
+      $result->error = "invalid schema $uri[scheme]";
+      return $result;
+  }
+
+  // Make sure the socket opened properly
+  if (!$fp) {
+    $result->error = trim("$errno $errstr");
+    return $result;
+  }
+
+  // Construct the path to act on
+  $path = $uri['path'] ? $uri['path'] : '/';
+  if ($uri['query']) {
+    $path .= "?$uri[query]";
+  }
+
+  // Create http request
+  $defaults = array(
+    'host' => "Host: $uri[host]",
+    'user-agent' => 'User-Agent: Drupal (+http://www.drupal.org/)',
+    'content-length' => 'Content-Length: '. strlen($data)
+  );
+
+  foreach ($headers as $header => $value) {
+    $defaults[$header] = "$header: $value";
+  }
+
+  $request = "$method $path HTTP/1.0\r\n";
+  $request .= implode("\r\n", $defaults);
+  $request .= "\r\n\r\n";
+  if ($data) {
+    $request .= "$data\r\n";
+  }
+  $result->request = $request;
+
+  fwrite($fp, $request);
+
+  // Fetch response.
+  while (!feof($fp) && $data = fgets($fp)) {
+    $response[] = $data;
+  }
+  fclose($fp);
+
+  // Parse response.
+  list($protocol, $code, $text) = explode(' ', trim(array_shift($response)), 3);
+  $result->headers = array();
+  $result->data = '';
+
+  // Parse headers.
+  while ($line = trim(array_shift($response))) {
+    if ($line == '') {
+      break;
+    }
+    list($header, $value) = explode(':', $line, 2);
+    $result->headers[$header] = trim($value);
+  }
+
+  $result->data = implode('', $response);
+
+  $responses = array(
+    100 => 'Continue', 101 => 'Switching Protocols',
+    200 => 'OK', 201 => 'Created', 202 => 'Accepted', 203 => 'Non-Authoritative Information', 204 => 'No Content', 205 => 'Reset Content', 206 => 'Partial Content',
+    300 => 'Multiple Choices', 301 => 'Moved Permanently', 302 => 'Found', 303 => 'See Other', 304 => 'Not Modified', 305 => 'Use Proxy', 307 => 'Temporary Redirect',
+    400 => 'Bad Request', 401 => 'Unauthorized', 402 => 'Payment Required', 403 => 'Forbidden', 404 => 'Not Found', 405 => 'Method Not Allowed', 406 => 'Not Acceptable', 407 => 'Proxy Authentication Required', 408 => 'Request Time-out', 409 => 'Conflict', 410 => 'Gone', 411 => 'Length Required', 412 => 'Precondition Failed', 413 => 'Request Entity Too Large', 414 => 'Request-URI Too Large', 415 => 'Unsupported Media Type', 416 => 'Requested range not satisfiable', 417 => 'Expectation Failed',
+    500 => 'Internal Server Error', 501 => 'Not Implemented', 502 => 'Bad Gateway', 503 => 'Service Unavailable', 504 => 'Gateway Time-out', 505 => 'HTTP Version not supported'
+  );
+  // RFC 2616 states that all unknown HTTP codes must be treated the same as
+  // the base code in their class:
+  if (!isset($responses[$code])) {
+    $code = floor($code / 100) * 100;
+  }
+
+  switch ($code) {
+    case 200: // OK
+    case 304: // Not modified
+      break;
+    case 301: // Moved permanently
+    case 302: // Moved temporarily
+    case 307: // Moved temporarily
+      $location = $result->headers['Location'];
+
+      if ($retry) {
+        $result = drupal_http_request($result->headers['Location'], $headers, $method, $data, --$retry);
+        $result->redirect_code = $result->code;
+      }
+      $result->redirect_url = $location;
+
+      break;
+    default:
+      $result->error = $text;
+  }
+
+  $result->code = $code;
+  return $result;
+}
 /* @} */
 
 function error_handler($errno, $message, $filename, $line, $variables) {
@@ -349,7 +470,7 @@ function drupal_specialchars($input, $quotes = ENT_NOQUOTES) {
  * allowed. See RFC 2822 for details.
  *
  * @param $mail A string containing an email address.
- * @return 
+ * @return
  */
 function valid_email_address($mail) {
   $user = '[a-zA-Z0-9_\-\.\+\^!#\$%&*+\/\=\?\`\|\{\}~\']+';

modules/drupal.module

@@ -36,7 +36,7 @@ function drupal_settings() {
     $error["drupal_directory"] = theme("error", t("You must set your site's slogan at the <a href=\"%url\">site configuration</a>.", array("%url" => url("admin/system"))));
   else if (variable_get("site_mission", "") == "")
     $error["drupal_directory"] = theme("error", t("You must set your site's mission at the <a href=\"%url\">site configuration</a>.", array("%url" => url("admin/system"))));
-  
+
   $output = form_textfield(t("Drupal XML-RPC server"), "drupal_server", variable_get("drupal_server", "http://www.drupal.org/xmlrpc.php"), 55, 128, t("The URL of your root Drupal XML-RPC server."));
   $output .= form_radios(t("Drupal directory"), "drupal_directory", variable_get("drupal_directory", 0), array(t("Disabled"), t("Enabled")), t("If enabled, your Drupal site will make itself known to the Drupal directory at the specified Drupal XML-RPC server. For this to work properly, you must set your site's name, e-mail address, slogan and mission statement. When the \"%xml-rpc-server\" field is set to \"%drupal-xml-rpc\", your web site will get listed on the %drupal-sites page. Requires the cron feature to be enabled.", array("%xml-rpc-server" => t("Drupal XML-RPC server"), "%drupal-xml-rpc" => "http://www.drupal.org/xmlrpc.php", "%drupal-sites" => "<a href=\"http://www.drupal.org/sites\">".t("Drupal sites")."</a>")) . $error["drupal_directory"]);
 

modules/drupal/drupal.module

@@ -36,7 +36,7 @@ function drupal_settings() {
     $error["drupal_directory"] = theme("error", t("You must set your site's slogan at the <a href=\"%url\">site configuration</a>.", array("%url" => url("admin/system"))));
   else if (variable_get("site_mission", "") == "")
     $error["drupal_directory"] = theme("error", t("You must set your site's mission at the <a href=\"%url\">site configuration</a>.", array("%url" => url("admin/system"))));
-  
+
   $output = form_textfield(t("Drupal XML-RPC server"), "drupal_server", variable_get("drupal_server", "http://www.drupal.org/xmlrpc.php"), 55, 128, t("The URL of your root Drupal XML-RPC server."));
   $output .= form_radios(t("Drupal directory"), "drupal_directory", variable_get("drupal_directory", 0), array(t("Disabled"), t("Enabled")), t("If enabled, your Drupal site will make itself known to the Drupal directory at the specified Drupal XML-RPC server. For this to work properly, you must set your site's name, e-mail address, slogan and mission statement. When the \"%xml-rpc-server\" field is set to \"%drupal-xml-rpc\", your web site will get listed on the %drupal-sites page. Requires the cron feature to be enabled.", array("%xml-rpc-server" => t("Drupal XML-RPC server"), "%drupal-xml-rpc" => "http://www.drupal.org/xmlrpc.php", "%drupal-sites" => "<a href=\"http://www.drupal.org/sites\">".t("Drupal sites")."</a>")) . $error["drupal_directory"]);
 

modules/filter.module

@@ -85,7 +85,7 @@ function filter_default($text) {
   }
 
   if (variable_get("filter_html", 0) == 2) {
-    // Escape HTML    
+    // Escape HTML
     $text = htmlspecialchars($text);
   }
 

modules/filter/filter.module

@@ -85,7 +85,7 @@ function filter_default($text) {
   }
 
   if (variable_get("filter_html", 0) == 2) {
-    // Escape HTML    
+    // Escape HTML
     $text = htmlspecialchars($text);
   }
 

modules/path.module

@@ -103,7 +103,6 @@ function path_form($edit = "", $error = "") {
 }
 
 function path_help($section = "admin/help#path") {
-  $output = "";
 
   switch ($section) {
     case "admin/system/modules#description":
@@ -116,10 +115,38 @@ function path_help($section = "admin/help#path") {
       $output = t("Enter the path you wish to create the alias for, followed by the name of the new alias. Each path can be associated with only one alias.");
       break;
     case "admin/help#path":
-      $output .= "<h3>Background</h3><p>A very powerful feature of Drupal is the ability to have control over all paths. The path module is the tool that provides this functionality and is part of the basic Drupal installation, although it is not enabled by default. Some examples of re-mapping paths are:<pre>user/login => login\n\nimage/tid/16 => store\n\ntaxonomy/page/or/7,19,20,21 => store/products/whirlygigs\n\nnode/view/3 => contact</pre></p>";
-      $output .= "<p>This functionality integrates seamlessly into node forms and also provides the administrator an interface to view all aliases that have been created.</p><p>Aliases have a 1 to 1 relationship with their original Drupal URLs. In other words you cannot have an alias map to more than one path. Likewise, a Drupal URL can't be mapped to more than one alias.</p>";
-      $output .= "<h3>Permissions</h3><p>Two new permissions are introduced for aliasing URLs: <i>create url aliases</i> and <i>administer url aliases</i>.</p><ol><li><strong>create url aliases</strong> - Allows users to create aliases for nodes. Enabling this permission will display a new path field to the user in any node form, allowing them to enter an alias for that node. They will be able to edit/delete the alias after it is created using the same form.</li><li><strong>administer url aliases</strong> - Allows users to access the alias administration interface.  They must also have the <i>access administration pages</i> permission set as well. This interface displays all aliases and provides a way to create and modify them. This is also the location to build aliases for things other than nodes.  For example, you can create an alias for a taxonomy URL or even re-map the admin path (although the original admin path will still be accessible since aliases do not cancel out original paths).</li></ol>";
-      $output = t($output);
+      $output = t("<h3>Background</h3>
+<p>A very powerful feature of Drupal is the ability to have control over all paths. The path module is the tool that provides this functionality and is part of the basic Drupal installation, although it is not enabled by default. Some examples of re-mapping paths are:</p>
+<pre>
+user/login => login
+
+image/tid/16 => store
+
+taxonomy/page/or/7,19,20,21 => store/products/whirlygigs
+
+node/view/3 => contact
+</pre>
+<p>This functionality integrates seamlessly into node forms and also provides the administrator an interface to view all aliases that have been created.</p>
+<p>Aliases have a 1 to 1 relationship with their original Drupal URLs. In other words you cannot have an alias map to more than one path. Likewise, a Drupal URL can't be mapped to more than one alias.</p>
+
+<h3>Permissions</h3>
+<p>Two permissions are related to URL aliasing: <i>create url aliases</i> and <i>administer url aliases</i>.</p>
+<ol><li><strong>create url aliases</strong> - Allows users to create aliases for nodes. Enabling this permission will display a path field to the user in any node form, allowing them to enter an alias for that node. They will be able to edit/delete the alias after it is created using the same form.</li><li><strong>administer url aliases</strong> - Allows users to access the alias administration interface. They must also have the <i>access administration pages</i> permission set as well. This interface displays all aliases and provides a way to create and modify them. This is also the location to build aliases for things other than nodes. For example, you can create an alias for a taxonomy URL or even re-map the admin path (although the original admin path will still be accessible since aliases do not cancel out original paths).</li></ol>
+
+<h3>Mass URL aliasing</h3>
+<p>Drupal also comes with user defined mass URL aliasing capabilities. You might like to see completely different URLs used by Drupal, or even URLs translated to the visitors' native language, in which case this feature is handy. Only an administrator with access to the website source code can set up this kind of aliases. You can define a <code>conf_url_rewrite</code> function in conf.php, following this example:</p>
+<pre>
+function conf_url_rewrite(\$path, \$mode = 'incoming') {
+  if (\$mode == 'incoming') { // URL coming from a client
+    return preg_replace('!^display/(\d+)$!', 'node/view/\1', \$path);
+  }
+  else { // URL going out to a client
+    return preg_replace('!^node/view/(\d+)$!', 'display/\1', \$path);
+  }
+}
+</pre>
+<p>This function will shorten every <code>node/view/\$node_id</code> type of URL to <code>display/\$node_id</code>. Individual URL aliases defined on the browser interface of Drupal take precedence, so if you have the 'contact' page alias from the example above, then the <code>display/3</code> alias will not be effective when outgoing links are created. Incoming URLs however always work with the mass URL aliased variant. Only the 'incoming' and 'outgoing' modes are supposed to be supported by your <code>conf_url_rewrite</code> function.</p>
+<p>You cannot only use this feature to shorten the URLs, or to translate them to you own language, but also to add completely new subURLs to an already existing module's URL space, or to compose a bunch of existing stuff together to a common URL space. You can create a <code>news</code> section for example aliasing nodes and taxonomy overview pages falling under a 'news' vocabulary, thus having <code>news/15</code> and <code>news/sections/3</code> instead of <code>node/view/15</code> and <code>taxonomy/view/or/3</code>.</p>");
       break;
   }
 

modules/path/path.module

@@ -103,7 +103,6 @@ function path_form($edit = "", $error = "") {
 }
 
 function path_help($section = "admin/help#path") {
-  $output = "";
 
   switch ($section) {
     case "admin/system/modules#description":
@@ -116,10 +115,38 @@ function path_help($section = "admin/help#path") {
       $output = t("Enter the path you wish to create the alias for, followed by the name of the new alias. Each path can be associated with only one alias.");
       break;
     case "admin/help#path":
-      $output .= "<h3>Background</h3><p>A very powerful feature of Drupal is the ability to have control over all paths. The path module is the tool that provides this functionality and is part of the basic Drupal installation, although it is not enabled by default. Some examples of re-mapping paths are:<pre>user/login => login\n\nimage/tid/16 => store\n\ntaxonomy/page/or/7,19,20,21 => store/products/whirlygigs\n\nnode/view/3 => contact</pre></p>";
-      $output .= "<p>This functionality integrates seamlessly into node forms and also provides the administrator an interface to view all aliases that have been created.</p><p>Aliases have a 1 to 1 relationship with their original Drupal URLs. In other words you cannot have an alias map to more than one path. Likewise, a Drupal URL can't be mapped to more than one alias.</p>";
-      $output .= "<h3>Permissions</h3><p>Two new permissions are introduced for aliasing URLs: <i>create url aliases</i> and <i>administer url aliases</i>.</p><ol><li><strong>create url aliases</strong> - Allows users to create aliases for nodes. Enabling this permission will display a new path field to the user in any node form, allowing them to enter an alias for that node. They will be able to edit/delete the alias after it is created using the same form.</li><li><strong>administer url aliases</strong> - Allows users to access the alias administration interface.  They must also have the <i>access administration pages</i> permission set as well. This interface displays all aliases and provides a way to create and modify them. This is also the location to build aliases for things other than nodes.  For example, you can create an alias for a taxonomy URL or even re-map the admin path (although the original admin path will still be accessible since aliases do not cancel out original paths).</li></ol>";
-      $output = t($output);
+      $output = t("<h3>Background</h3>
+<p>A very powerful feature of Drupal is the ability to have control over all paths. The path module is the tool that provides this functionality and is part of the basic Drupal installation, although it is not enabled by default. Some examples of re-mapping paths are:</p>
+<pre>
+user/login => login
+
+image/tid/16 => store
+
+taxonomy/page/or/7,19,20,21 => store/products/whirlygigs
+
+node/view/3 => contact
+</pre>
+<p>This functionality integrates seamlessly into node forms and also provides the administrator an interface to view all aliases that have been created.</p>
+<p>Aliases have a 1 to 1 relationship with their original Drupal URLs. In other words you cannot have an alias map to more than one path. Likewise, a Drupal URL can't be mapped to more than one alias.</p>
+
+<h3>Permissions</h3>
+<p>Two permissions are related to URL aliasing: <i>create url aliases</i> and <i>administer url aliases</i>.</p>
+<ol><li><strong>create url aliases</strong> - Allows users to create aliases for nodes. Enabling this permission will display a path field to the user in any node form, allowing them to enter an alias for that node. They will be able to edit/delete the alias after it is created using the same form.</li><li><strong>administer url aliases</strong> - Allows users to access the alias administration interface. They must also have the <i>access administration pages</i> permission set as well. This interface displays all aliases and provides a way to create and modify them. This is also the location to build aliases for things other than nodes. For example, you can create an alias for a taxonomy URL or even re-map the admin path (although the original admin path will still be accessible since aliases do not cancel out original paths).</li></ol>
+
+<h3>Mass URL aliasing</h3>
+<p>Drupal also comes with user defined mass URL aliasing capabilities. You might like to see completely different URLs used by Drupal, or even URLs translated to the visitors' native language, in which case this feature is handy. Only an administrator with access to the website source code can set up this kind of aliases. You can define a <code>conf_url_rewrite</code> function in conf.php, following this example:</p>
+<pre>
+function conf_url_rewrite(\$path, \$mode = 'incoming') {
+  if (\$mode == 'incoming') { // URL coming from a client
+    return preg_replace('!^display/(\d+)$!', 'node/view/\1', \$path);
+  }
+  else { // URL going out to a client
+    return preg_replace('!^node/view/(\d+)$!', 'display/\1', \$path);
+  }
+}
+</pre>
+<p>This function will shorten every <code>node/view/\$node_id</code> type of URL to <code>display/\$node_id</code>. Individual URL aliases defined on the browser interface of Drupal take precedence, so if you have the 'contact' page alias from the example above, then the <code>display/3</code> alias will not be effective when outgoing links are created. Incoming URLs however always work with the mass URL aliased variant. Only the 'incoming' and 'outgoing' modes are supposed to be supported by your <code>conf_url_rewrite</code> function.</p>
+<p>You cannot only use this feature to shorten the URLs, or to translate them to you own language, but also to add completely new subURLs to an already existing module's URL space, or to compose a bunch of existing stuff together to a common URL space. You can create a <code>news</code> section for example aliasing nodes and taxonomy overview pages falling under a 'news' vocabulary, thus having <code>news/15</code> and <code>news/sections/3</code> instead of <code>node/view/15</code> and <code>taxonomy/view/or/3</code>.</p>");
       break;
   }
 

update.php

@@ -63,7 +63,8 @@
   "2003-11-17" => "update_71",
   "2003-11-27" => "update_72",
   "2003-12-03" => "update_73",
-  "2003-12-06" => "update_74"
+  "2003-12-06" => "update_74",
+  "2004-01-06" => "update_75"
 );
 
 function update_32() {
@@ -590,6 +591,14 @@ function update_74() {
   update_sql("ALTER TABLE {system} ADD throttle tinyint(1) NOT NULL DEFAULT '0'");
 }
 
+function update_75() {
+  update_sql("ALTER TABLE {feed} ADD etag varchar(255) NOT NULL DEFAULT ''");
+  update_sql("ALTER TABLE {feed} ADD modified int(10) NOT NULL DEFAULT 0");
+  update_sql("ALTER TABLE {feed} CHANGE timestamp checked int(10) NOT NULL DEFAULT 0");
+  update_sql("UPDATE {blocks} SET module = 'aggregator' WHERE module = 'import'");
+  update_sql("UPDATE {system} SET filename = 'modules/aggregator.module', name = 'aggregator' WHERE filename = 'modules/import.module'");
+}
+
 /*
 ** System functions
 */