aggregator.module

<?php

function import_help() {
 ?>
  <P><I>TODO: introduction on syndication and a few pointers to more information.</I></P>
  <P>In Drupal you have <I>feeds</I> and <I>bundles</I>.   Feeds define news sources and bundles categoriz syndicated content by source, topic or any other heuristic.   Bundles provide a generalized way of creating composite feeds.  They allow you, for example, to combine various sport-related feeds into one bundle called "Sport".</P>
  <P>You can have several providers of news feeds.  You can add a feed by clicking the "add feed" link on the import administration pages.  Give the feed a name, supply the URI and a comma-separated list of attributes that you want to associate the feed with.  The update interval defines how often Drupal should go out to try and grab fresh content.  The expiration time defines how long syndicated content is kept in the database.  So set the update and expiration time and save your settings.  You have just defined your first feed.  If you have more feeds repeat as necessary.</P>
  <P>To verify whether your feed works, press "update items" at the overview page.  The number of items that have been sucessfully fetched, should then become visible in the third column of the feed overview.</P>
  <P>Now you have to define some bundles.  Bundles look for feeds that contain one of the keywords associated with the bundle and display those feeds together.  To define a bundle you have to give it a name and a comma-separated list of keywords just like this is the case for feeds.</P>
  <P>Your newly created bundle will now show up in the list of blocks that you can see at the block related administration pages.  There you can customize where and when your bundles will be displayed.</P>
 <?php
}

function import_perm() {
  return array("administer syndication");
}

function import_link($type) {
  if ($type == "admin") {
    $links[] = "<a href=\"admin.php?mod=import\">syndication</a>";
  }

  return $links ? $links : array();
}

function import_cron() {
  $result = db_query("SELECT * FROM feed");
  while ($feed = db_fetch_array($result)) {
    // remove expired items:
    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp < ". (time() - $feed[uncache]));

    // update feeds:
    if ($feed[timestamp] + $feed[refresh] < time()) import_update($feed);
  }
}

function import_bundle($attributes, $limit = 100) {
  if ($attributes) {
    // compose query:
    $keys = explode(",", $attributes);
    foreach ($keys as $key) $where[] = "attributes LIKE '%". trim($key) ."%'";

    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where) ." ORDER BY iid DESC LIMIT $limit");

    while ($item = db_fetch_object($result)) {
      $output .= "<li><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a></li>";
    }

    return "$output";
  }
}

function import_view_bundle() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $output .= "<b>$bundle->title</b><ul>". import_bundle($bundle->attributes) ."</ul>";
  }
  return $output;
}

function import_view_feed() {
  $result = db_query("SELECT * FROM feed ORDER BY title");
  while ($feed = db_fetch_object($result)) {
    $output .= "<b>$feed->title</b>". ($feed->link ? " (". format_url($feed->link) .")" : "") ."<ul>". check_output($feed->description) ."</ul>";
  }
  return $output;
}

function import_block() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $i++;
    $blocks[$i][subject] = $bundle->title;
    $blocks[$i][content] = import_bundle($bundle->attributes, 10);
    $blocks[$i][info] = "$bundle->title bundle";
  }
  return $blocks;
}

function import_remove($feed) {
  db_query("DELETE FROM item WHERE fid = '$feed[fid]'");
  return "feed '$feed[title]' reset.";
}

function import_update($feed) {

  // open socket:
  $url = parse_url($feed[url]);
  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);

  if ($fp) {
    // fetch data:
    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");

    // initialize the translation table:
    $tt = array_flip(get_html_translation_table(HTML_ENTITIES));
    $tt["&apos;"] = "'";

    while(!feof($fp)) $data .= fgets($fp, 128);

    if (strstr($data, "200 OK")) {
 
      /*
      ** Extract and process channel information:
      */

      $channel = ereg_replace("<item([^s].*)</item>", "", $data);
      $channel = ereg_replace("<image([^s].*)</image>", "", $channel);
      eregi("<title>(.*)</title>", $channel, $title);
      eregi("<link>(.*)</link>", $channel, $link);
      eregi("<description>(.*)</description>", $channel, $description);

      db_query("UPDATE feed SET timestamp = '". time() ."', link = '". check_input($link[1]) ."', description = '". check_input($description[1]) ."' WHERE fid = '". $feed[fid] ."'");

      /*
      ** Extract and process individual items:
      */

      eregi("<item([^s].*)</item>", $data, $data);

      $items = array_reverse(explode("</item>", $data[0]));

      foreach ($items as $item) {
        $t = eregi("<title>(.*)</title>", $item, $title);
        $l = eregi("<link>(.*)</link>", $item, $link);
        $a = eregi("<author>(.*)</author>", $item, $author);
        $d = eregi("<description>(.*)</description>", $item, $description);

        if ($l || $t || $a || $d) {
          $title = strtr(strip_tags($title[1]), $tt);
          $description = strtr($description[1], $tt);
          import_save_item(array(fid => $feed[fid], title => $title, link => $link[1], author => $author[1], description => $description, attributes => $feed[attributes]));
        }
      }
    }
    else {
      watchdog("error", "import: failed to syndicate from '$feed[title]'");
    }
  }

  return "feed '$feed[title]' updated.";
}

function import_save_item($edit) {
  if ($edit[iid] && $edit[title]) {
    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE iid = '$edit[iid]'");
  }
  else if ($edit[iid]) {
    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
  }
  else {
    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
      db_query("INSERT INTO item (fid, title, link, author, description, attributes, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attributes]) ."', '". time() ."')");
    }
  }
}

function import_form_bundle($edit = array()) {
  global $REQUEST_URI;

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the bundle.");

  $form .= form_submit("Submit");

  if ($edit[bid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("bid", $edit[bid]);
  }

  return form($REQUEST_URI, $form);
}

function import_save_bundle($edit) {
  if ($edit[bid] && $edit[title]) {
    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else if ($edit[bid]) {
    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else {
    db_query("INSERT INTO bundle (title, attributes) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attributes]) ."')");
  }

  module_rehash_blocks("import");
}

function import_form_feed($edit = array()) {
  global $REQUEST_URI;

  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
  $form .= form_textfield("URL", "url", $edit[url], 50, 128, "The fully-qualified URL of the feed.");
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the feed.");
  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");

  $form .= form_submit("Submit");

  if ($edit[fid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("fid", $edit[fid]);
  }

  return form($REQUEST_URI, $form);
}

function import_save_feed($edit) {
  if ($edit[fid] && $edit[title]) {
    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', url = '". check_input($edit[url]) ."', attributes = '". check_input($edit[attributes]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else if ($edit[fid]) {
    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else {
    db_query("INSERT INTO feed (title, url, attributes, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[url]) ."', '". check_input($edit[attributes]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
  }
}

function import_save_attributes($edit) {
  foreach($edit as $iid => $value) {
    db_query("UPDATE item SET attributes = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
  }
  return "attributes has been saved";
}

function import_get_feed($fid) {
  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
}

function import_get_bundle($bid) {
  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
}

function import_view() {
  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");

  $output .= "<h3>Feed overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>site</th><th>attributes</th><th>items</th><th>last update</th><th>next update</th><th colspan=\"3\">operations</th></tr>\n";
  while ($feed = db_fetch_object($result)) {
    $output .= " <tr><td>". check_output($feed->title) ."</td><td>". check_output($feed->attributes) ."</td><td>". format_plural($feed->items, "item", "items") ."</td><td>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</td><td>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</td><td><a href=\"admin.php?mod=import&type=feed&op=edit&id=$feed->fid\">edit feed</a></td><td><a href=\"admin.php?mod=import&type=feed&op=remove&id=$feed->fid\">remove items</a></td><td><a href=\"admin.php?mod=import&type=feed&op=update&id=$feed->fid\">update items</a></td></tr>\n";
  }
  $output .= "</table>\n";

  $result = db_query("SELECT * FROM bundle ORDER BY title");

  $output .= "<h3>Bundle overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>title</th><th>attributes</th><th>operations</th></tr>\n";
  while ($bundle = db_fetch_object($result)) {
    $output .= " <tr><td>". check_output($bundle->title) ."</td><td>". check_output($bundle->attributes) ."</td><td><a href=\"admin.php?mod=import&type=bundle&op=edit&id=$bundle->bid\">edit bundle</a></td></tr>\n";
  }
  $output .= "</table>\n";

  return $output;
}

function import_view_item() {
  global $REQUEST_URI;

  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");

  $output .= "<form action=\"$REQUEST_URI\" method=\"post\">\n";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>time</th><th>feed</th><th>item</th></tr>\n";
  while ($item = db_fetch_object($result)) {
    $output .= " <tr><td valign=\"top\" nowrap=\"nowrap\">". format_date($item->timestamp, "custom", "m/d/y") ."<br />".format_date($item->timestamp, "custom", "H:i") ."</td><td align=\"center\" valign=\"top\" nowrap=\"nowrap\"><a href=\"admin.php?mod=import&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</a></td><td><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a>". ($item->description ? "<br /><small><i>". check_output($item->description) ."</i></small>" : "") ."<br /><input type=\"text\" name=\"edit[$item->iid]\" value=\"". check_form($item->attributes) ."\" size=\"50\" /></td></tr>\n";
  }
  $output .= "</table>\n";
  $output .= "<input type=\"submit\" name=\"op\" value=\"Save attributes\" />\n";
  $output .= "</form>\n";

  return $output;
}

function import_admin() {
  global $op, $id, $type, $edit;

  if (user_access("administer syndication")) {

    print "<small><a href=\"admin.php?mod=import&type=feed&op=add\">add new feed</a> | <a href=\"admin.php?mod=import&type=bundle&op=add\">add new bundle</a> | <a href=\"admin.php?mod=import&type=feed&op=view\">available feeds</a> | <a href=\"admin.php?mod=import&type=bundle&op=view\">available bundles</a> | <a href=\"admin.php?mod=import&type=item&op=view\">available items</a> | <a href=\"admin.php?mod=import&op=view\">overview</a> | <a href=\"admin.php?mod=import&op=help\">help</a></small><hr />";

    switch($op) {
      case "help":
        print import_help();
        break;
      case "add":
        if ($type == "bundle")
          print import_form_bundle();
        else
          print import_form_feed();
        break;
      case "edit":
        if ($type == "bundle")
          print import_form_bundle(import_get_bundle($id));
        else
          print import_form_feed(import_get_feed($id));
        break;
      case "remove":
        print status(import_remove(import_get_feed($id)));
        print import_view_feed();
        break;
      case "update":
        print status(import_update(import_get_feed($id)));
        print import_view_feed();
        break;
      case "Save attributes":
        print status(import_save_attributes($edit));
        print import_view_item();
        break;
      case "Delete":
        $edit[title] = 0;
        // fall through:
      case "Submit":
        if ($type == "bundle")
          print status(import_save_bundle($edit));
        else
          print status(import_save_feed($edit));
        // fall through:
      default:
        if ($type == "bundle")
          print import_view_bundle();
        else if ($type == "feed")
          print import_view_feed();
        else if ($type == "item")
          print import_view_item();
        else
          print import_view();
    }
  }
  else {
    print message_access();
  }
}

?>