aggregator.module 15.5 KB
Newer Older
1
2
<?php

Dries's avatar
   
Dries committed
3
function import_help() {
4
 ?>
Dries's avatar
   
Dries committed
5
6
7
8
9
10
  <P><I>TODO: introduction on syndication and a few pointers to more information.</I></P>
  <P>In Drupal you have <I>feeds</I> and <I>bundles</I>.   Feeds define news sources and bundles categoriz syndicated content by source, topic or any other heuristic.   Bundles provide a generalized way of creating composite feeds.  They allow you, for example, to combine various sport-related feeds into one bundle called "Sport".</P>
  <P>You can have several providers of news feeds.  You can add a feed by clicking the "add feed" link on the import administration pages.  Give the feed a name, supply the URI and a comma-separated list of attributes that you want to associate the feed with.  The update interval defines how often Drupal should go out to try and grab fresh content.  The expiration time defines how long syndicated content is kept in the database.  So set the update and expiration time and save your settings.  You have just defined your first feed.  If you have more feeds repeat as necessary.</P>
  <P>To verify whether your feed works, press "update items" at the overview page.  The number of items that have been sucessfully fetched, should then become visible in the third column of the feed overview.</P>
  <P>Now you have to define some bundles.  Bundles look for feeds that contain one of the keywords associated with the bundle and display those feeds together.  To define a bundle you have to give it a name and a comma-separated list of keywords just like this is the case for feeds.</P>
  <P>Your newly created bundle will now show up in the list of blocks that you can see at the block related administration pages.  There you can customize where and when your bundles will be displayed.</P>
11
12
13
 <?php
}

Dries's avatar
   
Dries committed
14
function import_perm() {
Dries's avatar
   
Dries committed
15
  return array("administer syndication");
Dries's avatar
   
Dries committed
16
17
18
19
}

function import_link($type) {
  if ($type == "admin") {
Dries's avatar
   
Dries committed
20
    $links[] = "<a href=\"admin.php?mod=import\">syndication</a>";
Dries's avatar
   
Dries committed
21
22
23
  }

  return $links ? $links : array();
Dries's avatar
   
Dries committed
24
25
}

Dries's avatar
   
Dries committed
26
function import_cron() {
27
28
29
  $result = db_query("SELECT * FROM feed");
  while ($feed = db_fetch_array($result)) {
    // remove expired items:
Dries's avatar
   
Dries committed
30
    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp < ". (time() - $feed[uncache]));
31
32

    // update feeds:
Dries's avatar
   
Dries committed
33
    if ($feed[timestamp] + $feed[refresh] < time()) import_update($feed);
34
35
36
  }
}

37
38
function import_bundle($attributes, $limit = 100) {
  if ($attributes) {
39
    // compose query:
40
41
    $keys = explode(",", $attributes);
    foreach ($keys as $key) $where[] = "attributes LIKE '%". trim($key) ."%'";
42

Dries's avatar
   
Dries committed
43
    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where) ." ORDER BY iid DESC LIMIT $limit");
44
45

    while ($item = db_fetch_object($result)) {
Dries's avatar
   
Dries committed
46
      $output .= "<li><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a></li>";
47
48
    }

Dries's avatar
   
Dries committed
49
    return "$output";
50
51
52
  }
}

Dries's avatar
   
Dries committed
53
function import_view_bundle() {
Dries's avatar
   
Dries committed
54
  $result = db_query("SELECT * FROM bundle ORDER BY title");
55
  while ($bundle = db_fetch_object($result)) {
Dries's avatar
   
Dries committed
56
    $output .= "<b>$bundle->title</b><ul>". import_bundle($bundle->attributes) ."</ul>";
57
58
59
60
  }
  return $output;
}

Dries's avatar
   
Dries committed
61
62
63
64
65
66
67
68
function import_view_feed() {
  $result = db_query("SELECT * FROM feed ORDER BY title");
  while ($feed = db_fetch_object($result)) {
    $output .= "<b>$feed->title</b>". ($feed->link ? " (". format_url($feed->link) .")" : "") ."<ul>". check_output($feed->description) ."</ul>";
  }
  return $output;
}

Dries's avatar
   
Dries committed
69
70
71
72
73
function import_block() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $i++;
    $blocks[$i][subject] = $bundle->title;
74
    $blocks[$i][content] = import_bundle($bundle->attributes, 10);
Dries's avatar
   
Dries committed
75
76
77
78
79
    $blocks[$i][info] = "$bundle->title bundle";
  }
  return $blocks;
}

Dries's avatar
   
Dries committed
80
81
82
83
84
function import_remove($feed) {
  db_query("DELETE FROM item WHERE fid = '$feed[fid]'");
  return "feed '$feed[title]' reset.";
}

Dries's avatar
   
Dries committed
85
function import_update($feed) {
86
87

  // open socket:
Dries's avatar
   
Dries committed
88
  $url = parse_url($feed[url]);
89
90
91
92
93
  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);

  if ($fp) {
    // fetch data:
    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");
Dries's avatar
   
Dries committed
94
95
96
97
98

    // initialize the translation table:
    $tt = array_flip(get_html_translation_table(HTML_ENTITIES));
    $tt["&apos;"] = "'";

99
100
101
    while(!feof($fp)) $data .= fgets($fp, 128);

    if (strstr($data, "200 OK")) {
Dries's avatar
   
Dries committed
102
103
104
105
 
      /*
      ** Extract and process channel information:
      */
106

Dries's avatar
   
Dries committed
107
108
109
110
111
112
113
      $channel = ereg_replace("<item([^s].*)</item>", "", $data);
      $channel = ereg_replace("<image([^s].*)</image>", "", $channel);
      eregi("<title>(.*)</title>", $channel, $title);
      eregi("<link>(.*)</link>", $channel, $link);
      eregi("<description>(.*)</description>", $channel, $description);

      db_query("UPDATE feed SET timestamp = '". time() ."', link = '". check_input($link[1]) ."', description = '". check_input($description[1]) ."' WHERE fid = '". $feed[fid] ."'");
Dries's avatar
   
Dries committed
114

Dries's avatar
   
Dries committed
115
116
117
118
119
      /*
      ** Extract and process individual items:
      */

      eregi("<item([^s].*)</item>", $data, $data);
120

Dries's avatar
   
Dries committed
121
122
123
      $items = array_reverse(explode("</item>", $data[0]));

      foreach ($items as $item) {
124
        $t = eregi("<title>(.*)</title>", $item, $title);
Dries's avatar
   
Dries committed
125
        $l = eregi("<link>(.*)</link>", $item, $link);
126
127
128
129
        $a = eregi("<author>(.*)</author>", $item, $author);
        $d = eregi("<description>(.*)</description>", $item, $description);

        if ($l || $t || $a || $d) {
Dries's avatar
   
Dries committed
130
131
132
          $title = strtr(strip_tags($title[1]), $tt);
          $description = strtr($description[1], $tt);
          import_save_item(array(fid => $feed[fid], title => $title, link => $link[1], author => $author[1], description => $description, attributes => $feed[attributes]));
133
134
135
136
        }
      }
    }
    else {
Dries's avatar
   
Dries committed
137
      watchdog("error", "import: failed to syndicate from '$feed[title]'");
138
139
    }
  }
Dries's avatar
   
Dries committed
140
141

  return "feed '$feed[title]' updated.";
142
143
}

Dries's avatar
   
Dries committed
144
function import_save_item($edit) {
145
  if ($edit[iid] && $edit[title]) {
146
    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE iid = '$edit[iid]'");
147
148
149
150
151
152
  }
  else if ($edit[iid]) {
    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
  }
  else {
    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
153
      db_query("INSERT INTO item (fid, title, link, author, description, attributes, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attributes]) ."', '". time() ."')");
154
155
156
157
    }
  }
}

Dries's avatar
   
Dries committed
158
function import_form_bundle($edit = array()) {
159
160
161
  global $REQUEST_URI;

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
162
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the bundle.");
163
164
165
166
167
168
169
170
171
172
173

  $form .= form_submit("Submit");

  if ($edit[bid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("bid", $edit[bid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
   
Dries committed
174
function import_save_bundle($edit) {
175
  if ($edit[bid] && $edit[title]) {
176
    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
177
178
179
180
181
  }
  else if ($edit[bid]) {
    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else {
182
    db_query("INSERT INTO bundle (title, attributes) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attributes]) ."')");
183
  }
Dries's avatar
   
Dries committed
184
185

  module_rehash_blocks("import");
186
187
}

Dries's avatar
   
Dries committed
188
function import_form_feed($edit = array()) {
189
190
191
192
193
  global $REQUEST_URI;

  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
Dries's avatar
   
Dries committed
194
  $form .= form_textfield("URL", "url", $edit[url], 50, 128, "The fully-qualified URL of the feed.");
195
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the feed.");
196
197
198
199
200
201
202
203
204
205
206
207
208
  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");

  $form .= form_submit("Submit");

  if ($edit[fid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("fid", $edit[fid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
   
Dries committed
209
function import_save_feed($edit) {
210
  if ($edit[fid] && $edit[title]) {
Dries's avatar
   
Dries committed
211
    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', url = '". check_input($edit[url]) ."', attributes = '". check_input($edit[attributes]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
212
213
214
215
216
217
218
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else if ($edit[fid]) {
    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else {
Dries's avatar
   
Dries committed
219
    db_query("INSERT INTO feed (title, url, attributes, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[url]) ."', '". check_input($edit[attributes]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
220
221
222
  }
}

Dries's avatar
   
Dries committed
223
function import_save_attributes($edit) {
224
  foreach($edit as $iid => $value) {
225
    db_query("UPDATE item SET attributes = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
226
227
228
229
  }
  return "attributes has been saved";
}

Dries's avatar
   
Dries committed
230
function import_get_feed($fid) {
231
232
233
  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
}

Dries's avatar
   
Dries committed
234
function import_get_bundle($bid) {
235
236
237
  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
}

Dries's avatar
   
Dries committed
238
function import_view() {
239
240
  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");

Dries's avatar
   
Dries committed
241
242
  $output .= "<h3>Feed overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
Dries's avatar
   
Dries committed
243
  $output .= " <tr><th>site</th><th>attributes</th><th>items</th><th>last update</th><th>next update</th><th colspan=\"3\">operations</th></tr>\n";
244
  while ($feed = db_fetch_object($result)) {
Dries's avatar
   
Dries committed
245
    $output .= " <tr><td>". check_output($feed->title) ."</td><td>". check_output($feed->attributes) ."</td><td>". format_plural($feed->items, "item", "items") ."</td><td>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</td><td>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</td><td><a href=\"admin.php?mod=import&type=feed&op=edit&id=$feed->fid\">edit feed</a></td><td><a href=\"admin.php?mod=import&type=feed&op=remove&id=$feed->fid\">remove items</a></td><td><a href=\"admin.php?mod=import&type=feed&op=update&id=$feed->fid\">update items</a></td></tr>\n";
246
  }
Dries's avatar
   
Dries committed
247
  $output .= "</table>\n";
248
249
250

  $result = db_query("SELECT * FROM bundle ORDER BY title");

Dries's avatar
   
Dries committed
251
252
253
  $output .= "<h3>Bundle overview</h3>";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>title</th><th>attributes</th><th>operations</th></tr>\n";
254
  while ($bundle = db_fetch_object($result)) {
Dries's avatar
   
Dries committed
255
    $output .= " <tr><td>". check_output($bundle->title) ."</td><td>". check_output($bundle->attributes) ."</td><td><a href=\"admin.php?mod=import&type=bundle&op=edit&id=$bundle->bid\">edit bundle</a></td></tr>\n";
256
  }
Dries's avatar
   
Dries committed
257
  $output .= "</table>\n";
258
259
260
261

  return $output;
}

Dries's avatar
   
Dries committed
262
function import_view_item() {
263
264
265
266
  global $REQUEST_URI;

  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");

Dries's avatar
   
Dries committed
267
268
269
  $output .= "<form action=\"$REQUEST_URI\" method=\"post\">\n";
  $output .= "<table border=\"1\" cellpadding=\"2\" cellspacing=\"2\">\n";
  $output .= " <tr><th>time</th><th>feed</th><th>item</th></tr>\n";
270
  while ($item = db_fetch_object($result)) {
Dries's avatar
   
Dries committed
271
    $output .= " <tr><td valign=\"top\" nowrap=\"nowrap\">". format_date($item->timestamp, "custom", "m/d/y") ."<br />".format_date($item->timestamp, "custom", "H:i") ."</td><td align=\"center\" valign=\"top\" nowrap=\"nowrap\"><a href=\"admin.php?mod=import&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</a></td><td><a href=\"". check_output($item->link) ."\">". check_output($item->title) ."</a>". ($item->description ? "<br /><small><i>". check_output($item->description) ."</i></small>" : "") ."<br /><input type=\"text\" name=\"edit[$item->iid]\" value=\"". check_form($item->attributes) ."\" size=\"50\" /></td></tr>\n";
272
  }
Dries's avatar
   
Dries committed
273
274
275
  $output .= "</table>\n";
  $output .= "<input type=\"submit\" name=\"op\" value=\"Save attributes\" />\n";
  $output .= "</form>\n";
276
277
278
279

  return $output;
}

Dries's avatar
   
Dries committed
280
function import_admin() {
Dries's avatar
   
Dries committed
281
  global $op, $id, $type, $edit;
Dries's avatar
   
Dries committed
282

Dries's avatar
   
Dries committed
283
  if (user_access("administer syndication")) {
Dries's avatar
   
Dries committed
284

Dries's avatar
   
Dries committed
285
    print "<small><a href=\"admin.php?mod=import&type=feed&op=add\">add new feed</a> | <a href=\"admin.php?mod=import&type=bundle&op=add\">add new bundle</a> | <a href=\"admin.php?mod=import&type=feed&op=view\">available feeds</a> | <a href=\"admin.php?mod=import&type=bundle&op=view\">available bundles</a> | <a href=\"admin.php?mod=import&type=item&op=view\">available items</a> | <a href=\"admin.php?mod=import&op=view\">overview</a> | <a href=\"admin.php?mod=import&op=help\">help</a></small><hr />";
Dries's avatar
   
Dries committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308

    switch($op) {
      case "help":
        print import_help();
        break;
      case "add":
        if ($type == "bundle")
          print import_form_bundle();
        else
          print import_form_feed();
        break;
      case "edit":
        if ($type == "bundle")
          print import_form_bundle(import_get_bundle($id));
        else
          print import_form_feed(import_get_feed($id));
        break;
      case "remove":
        print status(import_remove(import_get_feed($id)));
        print import_view_feed();
        break;
      case "update":
        print status(import_update(import_get_feed($id)));
Dries's avatar
   
Dries committed
309
        print import_view_feed();
Dries's avatar
   
Dries committed
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
        break;
      case "Save attributes":
        print status(import_save_attributes($edit));
        print import_view_item();
        break;
      case "Delete":
        $edit[title] = 0;
        // fall through:
      case "Submit":
        if ($type == "bundle")
          print status(import_save_bundle($edit));
        else
          print status(import_save_feed($edit));
        // fall through:
      default:
        if ($type == "bundle")
          print import_view_bundle();
Dries's avatar
   
Dries committed
327
328
        else if ($type == "feed")
          print import_view_feed();
Dries's avatar
   
Dries committed
329
330
331
        else if ($type == "item")
          print import_view_item();
        else
Dries's avatar
   
Dries committed
332
          print import_view();
Dries's avatar
   
Dries committed
333
334
335
336
    }
  }
  else {
    print message_access();
337
338
339
340
  }
}

?>