aggregator.module 15.5 KB
Newer Older
1 2
<?php

Dries's avatar
 
Dries committed
3
function import_help() {
4
 ?>
Dries's avatar
 
Dries committed
5 6 7 8 9 10
  <P><I>TODO: introduction on syndication and a few pointers to more information.</I></P>
  <P>In Drupal you have <I>feeds</I> and <I>bundles</I>.   Feeds define news sources and bundles categoriz syndicated content by source, topic or any other heuristic.   Bundles provide a generalized way of creating composite feeds.  They allow you, for example, to combine various sport-related feeds into one bundle called "Sport".</P>
  <P>You can have several providers of news feeds.  You can add a feed by clicking the "add feed" link on the import administration pages.  Give the feed a name, supply the URI and a comma-separated list of attributes that you want to associate the feed with.  The update interval defines how often Drupal should go out to try and grab fresh content.  The expiration time defines how long syndicated content is kept in the database.  So set the update and expiration time and save your settings.  You have just defined your first feed.  If you have more feeds repeat as necessary.</P>
  <P>To verify whether your feed works, press "update items" at the overview page.  The number of items that have been sucessfully fetched, should then become visible in the third column of the feed overview.</P>
  <P>Now you have to define some bundles.  Bundles look for feeds that contain one of the keywords associated with the bundle and display those feeds together.  To define a bundle you have to give it a name and a comma-separated list of keywords just like this is the case for feeds.</P>
  <P>Your newly created bundle will now show up in the list of blocks that you can see at the block related administration pages.  There you can customize where and when your bundles will be displayed.</P>
11 12 13
 <?php
}

Dries's avatar
 
Dries committed
14
function import_perm() {
Dries's avatar
 
Dries committed
15
  return array("administer syndication");
Dries's avatar
 
Dries committed
16 17 18 19
}

function import_link($type) {
  if ($type == "admin") {
Dries's avatar
 
Dries committed
20
    $links[] = "<a href=\"admin.php?mod=import\">syndication</a>";
Dries's avatar
 
Dries committed
21 22 23
  }

  return $links ? $links : array();
Dries's avatar
 
Dries committed
24 25
}

Dries's avatar
 
Dries committed
26
function import_cron() {
27 28 29
  $result = db_query("SELECT * FROM feed");
  while ($feed = db_fetch_array($result)) {
    // remove expired items:
Dries's avatar
 
Dries committed
30
    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp < ". (time() - $feed[uncache]));
31 32

    // update feeds:
Dries's avatar
 
Dries committed
33
    if ($feed[timestamp] + $feed[refresh] < time()) import_update($feed);
34 35 36
  }
}

37 38
function import_bundle($attributes, $limit = 100) {
  if ($attributes) {
39
    // compose query:
40 41
    $keys = explode(",", $attributes);
    foreach ($keys as $key) $where[] = "attributes LIKE '%". trim($key) ."%'";
42

Dries's avatar
 
Dries committed
43
    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where) ." ORDER BY iid DESC LIMIT $limit");
44 45

    while ($item = db_fetch_object($result)) {
Dries's avatar
 
Dries committed
46
      $output .= "<li><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a></li>";
47 48
    }

Dries's avatar
 
Dries committed
49
    return "$output";
50 51 52
  }
}

Dries's avatar
 
Dries committed
53
function import_view_bundle() {
Dries's avatar
 
Dries committed
54
  $result = db_query("SELECT * FROM bundle ORDER BY title");
55
  while ($bundle = db_fetch_object($result)) {
Dries's avatar
 
Dries committed
56
    $output .= "<b>$bundle->title</b><ul>". import_bundle($bundle->attributes) ."</ul>";
57 58 59 60
  }
  return $output;
}

Dries's avatar
 
Dries committed
61 62 63 64 65 66 67 68
function import_view_feed() {
  $result = db_query("SELECT * FROM feed ORDER BY title");
  while ($feed = db_fetch_object($result)) {
    $output .= "<b>$feed->title</b>". ($feed->link ? " (". format_url($feed->link) .")" : "") ."<ul>". check_output($feed->description) ."</ul>";
  }
  return $output;
}

Dries's avatar
 
Dries committed
69 70 71 72 73
function import_block() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $i++;
    $blocks[$i][subject] = $bundle->title;
74
    $blocks[$i][content] = import_bundle($bundle->attributes, 10);
Dries's avatar
 
Dries committed
75 76 77 78 79
    $blocks[$i][info] = "$bundle->title bundle";
  }
  return $blocks;
}

Dries's avatar
 
Dries committed
80 81 82 83 84
function import_remove($feed) {
  db_query("DELETE FROM item WHERE fid = '$feed[fid]'");
  return "feed '$feed[title]' reset.";
}

Dries's avatar
 
Dries committed
85
function import_update($feed) {
86 87

  // open socket:
Dries's avatar
 
Dries committed
88
  $url = parse_url($feed[url]);
89 90 91 92 93
  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);

  if ($fp) {
    // fetch data:
    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");
Dries's avatar
 
Dries committed
94 95 96 97 98

    // initialize the translation table:
    $tt = array_flip(get_html_translation_table(HTML_ENTITIES));
    $tt["&apos;"] = "'";

99 100 101
    while(!feof($fp)) $data .= fgets($fp, 128);

    if (strstr($data, "200 OK")) {
Dries's avatar
 
Dries committed
102 103 104 105
 
      /*
      ** Extract and process channel information:
      */
106

Dries's avatar
 
Dries committed
107 108 109 110 111 112 113
      $channel = ereg_replace("<item([^s].*)</item>", "", $data);
      $channel = ereg_replace("<image([^s].*)</image>", "", $channel);
      eregi("<title>(.*)</title>", $channel, $title);
      eregi("<link>(.*)</link>", $channel, $link);
      eregi("<description>(.*)</description>", $channel, $description);

      db_query("UPDATE feed SET timestamp = '". time() ."', link = '". check_input($link[1]) ."', description = '". check_input($description[1]) ."' WHERE fid = '". $feed[fid] ."'");
Dries's avatar
 
Dries committed
114

Dries's avatar
 
Dries committed
115 116 117 118 119
      /*
      ** Extract and process individual items:
      */

      eregi("<item([^s].*)</item>", $data, $data);
120

Dries's avatar
 
Dries committed
121 122 123
      $items = array_reverse(explode("</item>", $data[0]));

      foreach ($items as $item) {
124
        $t = eregi("<title>(.*)</title>", $item, $title);
Dries's avatar
 
Dries committed
125
        $l = eregi("<link>(.*)</link>", $item, $link);
126 127 128 129
        $a = eregi("<author>(.*)</author>", $item, $author);
        $d = eregi("<description>(.*)</description>", $item, $description);

        if ($l || $t || $a || $d) {
Dries's avatar
 
Dries committed
130 131 132
          $title = strtr(strip_tags($title[1]), $tt);
          $description = strtr($description[1], $tt);
          import_save_item(array(fid => $feed[fid], title => $title, link => $link[1], author => $author[1], description => $description, attributes => $feed[attributes]));
133 134 135 136
        }
      }
    }
    else {
Dries's avatar
 
Dries committed
137
      watchdog("error", "import: failed to syndicate from '$feed[title]'");
138 139
    }
  }
Dries's avatar
 
Dries committed
140 141

  return "feed '$feed[title]' updated.";
142 143
}

Dries's avatar
 
Dries committed
144
function import_save_item($edit) {
145
  if ($edit[iid] && $edit[title]) {
146
    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE iid = '$edit[iid]'");
147 148 149 150 151 152
  }
  else if ($edit[iid]) {
    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
  }
  else {
    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
153
      db_query("INSERT INTO item (fid, title, link, author, description, attributes, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attributes]) ."', '". time() ."')");
154 155 156 157
    }
  }
}

Dries's avatar
 
Dries committed
158
function import_form_bundle($edit = array()) {
159 160 161
  global $REQUEST_URI;

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
162
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the bundle.");
163 164 165 166 167 168 169 170 171 172 173

  $form .= form_submit("Submit");

  if ($edit[bid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("bid", $edit[bid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
 
Dries committed
174
function import_save_bundle($edit) {
175
  if ($edit[bid] && $edit[title]) {
176
    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
177 178 179 180 181
  }
  else if ($edit[bid]) {
    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else {
182
    db_query("INSERT INTO bundle (title, attributes) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attributes]) ."')");
183
  }
Dries's avatar
 
Dries committed
184 185

  module_rehash_blocks("import");
186 187
}

Dries's avatar
 
Dries committed
188
function import_form_feed($edit = array()) {
189 190 191 192 193
  global $REQUEST_URI;

  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
Dries's avatar
 
Dries committed
194
  $form .= form_textfield("URL", "url", $edit[url], 50, 128, "The fully-qualified URL of the feed.");
195
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the feed.");
196 197 198 199 200 201 202 203 204 205 206 207 208
  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");

  $form .= form_submit("Submit");

  if ($edit[fid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("fid", $edit[fid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
 
Dries committed
209
function import_save_feed($edit) {
210
  if ($edit[fid] && $edit[title]) {
Dries's avatar
 
Dries committed
211
    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', url = '". check_input($edit[url]) ."', attributes = '". check_input($edit[attributes]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
212 213 214 215 216 217 218
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else if ($edit[fid]) {
    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else {
Dries's avatar
 
Dries committed
219
    db_query("INSERT INTO feed (title, url, attributes, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[url]) ."', '". check_input($edit[attributes]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
220 221 222
  }
}

Dries's avatar
 
Dries committed
223
function import_save_attributes($edit) {
224
  foreach($edit as $iid => $value) {
225
    db_query("UPDATE item SET attributes = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
226 227 228 229
  }
  return "attributes has been saved";
}

Dries's avatar
 
Dries committed
230
function import_get_feed($fid) {
231 232 233
  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
}

Dries's avatar
 
Dries committed
234
function import_get_bundle($bid) {
235 236 237
  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
}

Dries's avatar
 
Dries committed
238
function import_view() {
239 240
  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");

Dries's avatar
 
Dries committed
241 242
  $output .= "<h3>Feed overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
Dries's avatar
 
Dries committed
243
  $output .= " <tr><th>site</th><th>attributes</th><th>items</th><th>last update</th><th>next update</th><th colspan=\"3\">operations</th></tr>\n";
244
  while ($feed = db_fetch_object($result)) {
Dries's avatar
 
Dries committed
245
    $output .= " <tr><td>". check_output($feed->title) ."</td><td>". check_output($feed->attributes) ."</td><td>". format_plural($feed->items, "item", "items") ."</td><td>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</td><td>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</td><td><a href=\"admin.php?mod=import&type=feed&op=edit&id=$feed->fid\">edit feed</a></td><td><a href=\"admin.php?mod=import&type=feed&op=remove&id=$feed->fid\">remove items</a></td><td><a href=\"admin.php?mod=import&type=feed&op=update&id=$feed->fid\">update items</a></td></tr>\n";
246
  }
Dries's avatar
 
Dries committed
247
  $output .= "</table>\n";
248 249 250

  $result = db_query("SELECT * FROM bundle ORDER BY title");

Dries's avatar
 
Dries committed
251 252 253
  $output .= "<h3>Bundle overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>title</th><th>attributes</th><th>operations</th></tr>\n";
254
  while ($bundle = db_fetch_object($result)) {
Dries's avatar
 
Dries committed
255
    $output .= " <tr><td>". check_output($bundle->title) ."</td><td>". check_output($bundle->attributes) ."</td><td><a href=\"admin.php?mod=import&type=bundle&op=edit&id=$bundle->bid\">edit bundle</a></td></tr>\n";
256
  }
Dries's avatar
 
Dries committed
257
  $output .= "</table>\n";
258 259 260 261

  return $output;
}

Dries's avatar
 
Dries committed
262
function import_view_item() {
263 264 265 266
  global $REQUEST_URI;

  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");

Dries's avatar
 
Dries committed
267 268 269
  $output .= "<form action=\"$REQUEST_URI\" method=\"post\">\n";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>time</th><th>feed</th><th>item</th></tr>\n";
270
  while ($item = db_fetch_object($result)) {
Dries's avatar
 
Dries committed
271
    $output .= " <tr><td valign=\"top\" nowrap=\"nowrap\">". format_date($item->timestamp, "custom", "m/d/y") ."<br />".format_date($item->timestamp, "custom", "H:i") ."</td><td align=\"center\" valign=\"top\" nowrap=\"nowrap\"><a href=\"admin.php?mod=import&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</a></td><td><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a>". ($item->description ? "<br /><small><i>". check_output($item->description) ."</i></small>" : "") ."<br /><input type=\"text\" name=\"edit[$item->iid]\" value=\"". check_form($item->attributes) ."\" size=\"50\" /></td></tr>\n";
272
  }
Dries's avatar
 
Dries committed
273 274 275
  $output .= "</table>\n";
  $output .= "<input type=\"submit\" name=\"op\" value=\"Save attributes\" />\n";
  $output .= "</form>\n";
276 277 278 279

  return $output;
}

Dries's avatar
 
Dries committed
280
function import_admin() {
Dries's avatar
 
Dries committed
281
  global $op, $id, $type, $edit;
Dries's avatar
 
Dries committed
282

Dries's avatar
 
Dries committed
283
  if (user_access("administer syndication")) {
Dries's avatar
 
Dries committed
284

Dries's avatar
 
Dries committed
285
    print "<small><a href=\"admin.php?mod=import&type=feed&op=add\">add new feed</a> | <a href=\"admin.php?mod=import&type=bundle&op=add\">add new bundle</a> | <a href=\"admin.php?mod=import&type=feed&op=view\">available feeds</a> | <a href=\"admin.php?mod=import&type=bundle&op=view\">available bundles</a> | <a href=\"admin.php?mod=import&type=item&op=view\">available items</a> | <a href=\"admin.php?mod=import&op=view\">overview</a> | <a href=\"admin.php?mod=import&op=help\">help</a></small><hr />";
Dries's avatar
 
Dries committed
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308

    switch($op) {
      case "help":
        print import_help();
        break;
      case "add":
        if ($type == "bundle")
          print import_form_bundle();
        else
          print import_form_feed();
        break;
      case "edit":
        if ($type == "bundle")
          print import_form_bundle(import_get_bundle($id));
        else
          print import_form_feed(import_get_feed($id));
        break;
      case "remove":
        print status(import_remove(import_get_feed($id)));
        print import_view_feed();
        break;
      case "update":
        print status(import_update(import_get_feed($id)));
Dries's avatar
 
Dries committed
309
        print import_view_feed();
Dries's avatar
 
Dries committed
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
        break;
      case "Save attributes":
        print status(import_save_attributes($edit));
        print import_view_item();
        break;
      case "Delete":
        $edit[title] = 0;
        // fall through:
      case "Submit":
        if ($type == "bundle")
          print status(import_save_bundle($edit));
        else
          print status(import_save_feed($edit));
        // fall through:
      default:
        if ($type == "bundle")
          print import_view_bundle();
Dries's avatar
 
Dries committed
327 328
        else if ($type == "feed")
          print import_view_feed();
Dries's avatar
 
Dries committed
329 330 331
        else if ($type == "item")
          print import_view_item();
        else
Dries's avatar
 
Dries committed
332
          print import_view();
Dries's avatar
 
Dries committed
333 334 335 336
    }
  }
  else {
    print message_access();
337 338 339 340
  }
}

?>