aggregator.module 14.2 KB
Newer Older
1 2
<?php

Dries's avatar
 
Dries committed
3
function import_help() {
4
 ?>
Dries's avatar
 
Dries committed
5 6 7 8 9 10
  <P><I>TODO: introduction on syndication and a few pointers to more information.</I></P>
  <P>In Drupal you have <I>feeds</I> and <I>bundles</I>.   Feeds define news sources and bundles categoriz syndicated content by source, topic or any other heuristic.   Bundles provide a generalized way of creating composite feeds.  They allow you, for example, to combine various sport-related feeds into one bundle called "Sport".</P>
  <P>You can have several providers of news feeds.  You can add a feed by clicking the "add feed" link on the import administration pages.  Give the feed a name, supply the URI and a comma-separated list of attributes that you want to associate the feed with.  The update interval defines how often Drupal should go out to try and grab fresh content.  The expiration time defines how long syndicated content is kept in the database.  So set the update and expiration time and save your settings.  You have just defined your first feed.  If you have more feeds repeat as necessary.</P>
  <P>To verify whether your feed works, press "update items" at the overview page.  The number of items that have been sucessfully fetched, should then become visible in the third column of the feed overview.</P>
  <P>Now you have to define some bundles.  Bundles look for feeds that contain one of the keywords associated with the bundle and display those feeds together.  To define a bundle you have to give it a name and a comma-separated list of keywords just like this is the case for feeds.</P>
  <P>Your newly created bundle will now show up in the list of blocks that you can see at the block related administration pages.  There you can customize where and when your bundles will be displayed.</P>
11 12 13
 <?php
}

Dries's avatar
 
Dries committed
14 15 16 17
function import_perm() {
  return array("add and edit news feeds");
}

Dries's avatar
 
Dries committed
18
function import_cron() {
19 20 21
  $result = db_query("SELECT * FROM feed");
  while ($feed = db_fetch_array($result)) {
    // remove expired items:
Dries's avatar
 
Dries committed
22
    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp < ". (time() - $feed[uncache]));
23 24

    // update feeds:
Dries's avatar
 
Dries committed
25
    if ($feed[timestamp] + $feed[refresh] < time()) import_update($feed);
26 27 28
  }
}

29 30
function import_bundle($attributes, $limit = 100) {
  if ($attributes) {
31
    // compose query:
32 33
    $keys = explode(",", $attributes);
    foreach ($keys as $key) $where[] = "attributes LIKE '%". trim($key) ."%'";
34

Dries's avatar
 
Dries committed
35
    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where) ." ORDER BY timestamp DESC LIMIT $limit");
36 37 38 39 40

    while ($item = db_fetch_object($result)) {
      $output .= "<LI><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A></LI>";
    }

Dries's avatar
 
Dries committed
41
    return "$output";
42 43 44
  }
}

Dries's avatar
 
Dries committed
45
function import_view_bundle() {
Dries's avatar
 
Dries committed
46
  $result = db_query("SELECT * FROM bundle ORDER BY title");
47
  while ($bundle = db_fetch_object($result)) {
48
    $output .= "<B>$bundle->title</B><UL>". import_bundle($bundle->attributes) ."</UL>";
49 50 51 52
  }
  return $output;
}

Dries's avatar
 
Dries committed
53 54 55 56 57
function import_block() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $i++;
    $blocks[$i][subject] = $bundle->title;
58
    $blocks[$i][content] = import_bundle($bundle->attributes, 10);
Dries's avatar
 
Dries committed
59 60 61 62 63
    $blocks[$i][info] = "$bundle->title bundle";
  }
  return $blocks;
}

Dries's avatar
 
Dries committed
64 65 66 67 68
function import_remove($feed) {
  db_query("DELETE FROM item WHERE fid = '$feed[fid]'");
  return "feed '$feed[title]' reset.";
}

Dries's avatar
 
Dries committed
69
function import_update($feed) {
70 71 72 73 74 75 76 77 78 79 80 81

  // open socket:
  $url = parse_url($feed[link]);
  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);

  if ($fp) {
    // fetch data:
    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");
    while(!feof($fp)) $data .= fgets($fp, 128);

    if (strstr($data, "200 OK")) {

Dries's avatar
 
Dries committed
82 83 84
      eregi("<item([^s].*)</item>", $data, $data);

      // print "<PRE>". htmlentities($data[0]) ."</PRE>";
85 86 87

      foreach (explode("</item>", $data[0]) as $item) {
        $t = eregi("<title>(.*)</title>", $item, $title);
Dries's avatar
 
Dries committed
88
        $l = eregi("<link>(.*)</link>", $item, $link);
89 90 91 92
        $a = eregi("<author>(.*)</author>", $item, $author);
        $d = eregi("<description>(.*)</description>", $item, $description);

        if ($l || $t || $a || $d) {
93
          import_save_item(array(fid => $feed[fid], title => $title[0], link => $link[0], author => $author[0], description => $description[0], attributes => $feed[attributes]));
94 95 96 97 98 99 100 101 102
        }
      }

      db_query("UPDATE feed SET timestamp = '". time() ."' WHERE fid = '". $feed[fid] ."'");
    }
    else {
      watchdog("error", "failed to syndicate from '$feed[title]'");
    }
  }
Dries's avatar
 
Dries committed
103 104

  return "feed '$feed[title]' updated.";
105 106
}

Dries's avatar
 
Dries committed
107
function import_save_item($edit) {
108
  if ($edit[iid] && $edit[title]) {
109
    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE iid = '$edit[iid]'");
110 111 112 113 114 115
  }
  else if ($edit[iid]) {
    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
  }
  else {
    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
116
      db_query("INSERT INTO item (fid, title, link, author, description, attributes, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attributes]) ."', '". time() ."')");
117 118 119 120
    }
  }
}

Dries's avatar
 
Dries committed
121
function import_form_bundle($edit = array()) {
122 123 124
  global $REQUEST_URI;

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
125
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the bundle.");
126 127 128 129 130 131 132 133 134 135 136

  $form .= form_submit("Submit");

  if ($edit[bid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("bid", $edit[bid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
 
Dries committed
137
function import_save_bundle($edit) {
138
  if ($edit[bid] && $edit[title]) {
139
    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
140 141 142 143 144
  }
  else if ($edit[bid]) {
    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else {
145
    db_query("INSERT INTO bundle (title, attributes) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attributes]) ."')");
146
  }
Dries's avatar
 
Dries committed
147 148

  module_rehash_blocks("import");
149 150
}

Dries's avatar
 
Dries committed
151
function import_form_feed($edit = array()) {
152 153 154 155 156
  global $REQUEST_URI;

  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
Dries's avatar
 
Dries committed
157
  $form .= form_textfield("Link", "link", $edit[link], 50, 128, "The fully-qualified URL of the feed.");
158
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the feed.");
159 160 161 162 163 164 165 166 167 168 169 170 171
  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");

  $form .= form_submit("Submit");

  if ($edit[fid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("fid", $edit[fid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
 
Dries committed
172
function import_save_feed($edit) {
173
  if ($edit[fid] && $edit[title]) {
174
    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', attributes = '". check_input($edit[attributes]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
175 176 177 178 179 180 181
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else if ($edit[fid]) {
    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else {
182
    db_query("INSERT INTO feed (title, link, attributes, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[attributes]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
183 184 185
  }
}

Dries's avatar
 
Dries committed
186
function import_save_attributes($edit) {
187
  foreach($edit as $iid => $value) {
188
    db_query("UPDATE item SET attributes = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
189 190 191 192
  }
  return "attributes has been saved";
}

Dries's avatar
 
Dries committed
193
function import_get_feed($fid) {
194 195 196
  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
}

Dries's avatar
 
Dries committed
197
function import_get_bundle($bid) {
198 199 200
  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
}

Dries's avatar
 
Dries committed
201
function import_view_feed() {
202 203
  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");

Dries's avatar
 
Dries committed
204
  $output .= "<H3>Feed overview</H3>";
205
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
Dries's avatar
 
Dries committed
206
  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>items</TH><TH>last update</TH><TH>next update</TH><TH COLSPAN=\"3\">operations</TH></TR>\n";
207
  while ($feed = db_fetch_object($result)) {
208
    $output .= " <TR><TD>". check_output($feed->title) ."</TD><TD>". check_output($feed->attributes) ."</TD><TD>". format_plural($feed->items, "item", "items") ."</TD><TD>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</TD><TD>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=edit&id=$feed->fid\">edit feed</A></TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=remove&id=$feed->fid\">remove items</A></TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=update&id=$feed->fid\">update items</A></TD></TR>\n";
209 210 211 212 213
  }
  $output .= "</TABLE>\n";

  $result = db_query("SELECT * FROM bundle ORDER BY title");

Dries's avatar
 
Dries committed
214
  $output .= "<H3>Bundle overview</H3>";
215 216 217
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>operations</TH></TR>\n";
  while ($bundle = db_fetch_object($result)) {
218
    $output .= " <TR><TD>". check_output($bundle->title) ."</TD><TD>". check_output($bundle->attributes) ."</TD><TD><A HREF=\"admin.php?mod=import&type=bundle&op=edit&id=$bundle->bid\">edit bundle</A></TD></TR>\n";
219 220 221 222 223 224
  }
  $output .= "</TABLE>\n";

  return $output;
}

Dries's avatar
 
Dries committed
225
function import_view_item() {
226 227 228 229 230 231 232 233
  global $REQUEST_URI;

  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");

  $output .= "<FORM ACTION=\"$REQUEST_URI\" METHOD=\"post\">\n";
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
  $output .= " <TR><TH>time</TH><TH>feed</TH><TH>item</TH></TR>\n";
  while ($item = db_fetch_object($result)) {
234
    $output .= " <TR><TD VALIGN=\"top\" NOWRAP>". format_date($item->timestamp, "custom", "m/d/y") ."<BR>".format_date($item->timestamp, "custom", "H:i") ."</TD><TD ALIGN=\"center\" VALIGN=\"top\" NOWRAP><A HREF=\"admin.php?mod=import&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</A></TD><TD><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A>". ($item->description ? "<BR><SMALL><I>". check_output($item->description) ."</I></SMALL>" : "") ."<BR><INPUT TYPE=\"text\" NAME=\"edit[$item->iid]\" VALUE=\"". check_form($item->attributes) ."\" SIZE=\"50\"></TD></TR>\n";
235 236 237 238 239 240 241 242
  }
  $output .= "</TABLE>\n";
  $output .= "<INPUT TYPE=\"submit\" NAME=\"op\" VALUE=\"Save attributes\">\n";
  $output .= "</FORM>\n";

  return $output;
}

Dries's avatar
 
Dries committed
243
function import_admin() {
Dries's avatar
 
Dries committed
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
  global $user, $op, $id, $type, $edit;

  if (user_access($user, "add and edit news feeds")) {

    print "<SMALL><A HREF=\"admin.php?mod=import&type=bundle&op=add\">add new bundle</A> | <A HREF=\"admin.php?mod=import&type=feed&op=add\">add new feed</A> | <A HREF=\"admin.php?mod=import&type=bundle&op=view\">available bundles</A> | <A HREF=\"admin.php?mod=import&type=item&op=view\">available items</A> | <A HREF=\"admin.php?mod=import&op=view\">overview</A> | <A HREF=\"admin.php?mod=import&op=help\">help</A></SMALL><HR>";

    switch($op) {
      case "help":
        print import_help();
        break;
      case "add":
        if ($type == "bundle")
          print import_form_bundle();
        else
          print import_form_feed();
        break;
      case "edit":
        if ($type == "bundle")
          print import_form_bundle(import_get_bundle($id));
        else
          print import_form_feed(import_get_feed($id));
        break;
      case "remove":
        print status(import_remove(import_get_feed($id)));
        print import_view_feed();
        break;
      case "update":
        print status(import_update(import_get_feed($id)));
Dries's avatar
 
Dries committed
272
        print import_view_feed();
Dries's avatar
 
Dries committed
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
        break;
      case "Save attributes":
        print status(import_save_attributes($edit));
        print import_view_item();
        break;
      case "Delete":
        $edit[title] = 0;
        // fall through:
      case "Submit":
        if ($type == "bundle")
          print status(import_save_bundle($edit));
        else
          print status(import_save_feed($edit));
        // fall through:
      default:
        if ($type == "bundle")
          print import_view_bundle();
        else if ($type == "item")
          print import_view_item();
        else
          print import_view_feed();
    }
  }
  else {
    print message_access();
298 299 300 301
  }
}

?>