aggregator.module 14.2 KB
Newer Older
1
2
<?php

Dries's avatar
   
Dries committed
3
function import_help() {
4
 ?>
Dries's avatar
   
Dries committed
5
6
7
8
9
10
  <P><I>TODO: introduction on syndication and a few pointers to more information.</I></P>
  <P>In Drupal you have <I>feeds</I> and <I>bundles</I>.   Feeds define news sources and bundles categoriz syndicated content by source, topic or any other heuristic.   Bundles provide a generalized way of creating composite feeds.  They allow you, for example, to combine various sport-related feeds into one bundle called "Sport".</P>
  <P>You can have several providers of news feeds.  You can add a feed by clicking the "add feed" link on the import administration pages.  Give the feed a name, supply the URI and a comma-separated list of attributes that you want to associate the feed with.  The update interval defines how often Drupal should go out to try and grab fresh content.  The expiration time defines how long syndicated content is kept in the database.  So set the update and expiration time and save your settings.  You have just defined your first feed.  If you have more feeds repeat as necessary.</P>
  <P>To verify whether your feed works, press "update items" at the overview page.  The number of items that have been sucessfully fetched, should then become visible in the third column of the feed overview.</P>
  <P>Now you have to define some bundles.  Bundles look for feeds that contain one of the keywords associated with the bundle and display those feeds together.  To define a bundle you have to give it a name and a comma-separated list of keywords just like this is the case for feeds.</P>
  <P>Your newly created bundle will now show up in the list of blocks that you can see at the block related administration pages.  There you can customize where and when your bundles will be displayed.</P>
11
12
13
 <?php
}

Dries's avatar
   
Dries committed
14
15
16
17
function import_perm() {
  return array("add and edit news feeds");
}

Dries's avatar
   
Dries committed
18
function import_cron() {
19
20
21
  $result = db_query("SELECT * FROM feed");
  while ($feed = db_fetch_array($result)) {
    // remove expired items:
Dries's avatar
   
Dries committed
22
    db_query("DELETE FROM item WHERE fid = '$feed[fid]' AND timestamp < ". (time() - $feed[uncache]));
23
24

    // update feeds:
Dries's avatar
   
Dries committed
25
    if ($feed[timestamp] + $feed[refresh] < time()) import_update($feed);
26
27
28
  }
}

29
30
function import_bundle($attributes, $limit = 100) {
  if ($attributes) {
31
    // compose query:
32
33
    $keys = explode(",", $attributes);
    foreach ($keys as $key) $where[] = "attributes LIKE '%". trim($key) ."%'";
34

Dries's avatar
   
Dries committed
35
    $result = db_query("SELECT * FROM item WHERE ". implode(" OR ", $where) ." ORDER BY timestamp DESC LIMIT $limit");
36
37
38
39
40

    while ($item = db_fetch_object($result)) {
      $output .= "<LI><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A></LI>";
    }

Dries's avatar
   
Dries committed
41
    return "$output";
42
43
44
  }
}

Dries's avatar
   
Dries committed
45
function import_view_bundle() {
Dries's avatar
   
Dries committed
46
  $result = db_query("SELECT * FROM bundle ORDER BY title");
47
  while ($bundle = db_fetch_object($result)) {
48
    $output .= "<B>$bundle->title</B><UL>". import_bundle($bundle->attributes) ."</UL>";
49
50
51
52
  }
  return $output;
}

Dries's avatar
   
Dries committed
53
54
55
56
57
function import_block() {
  $result = db_query("SELECT * FROM bundle ORDER BY title");
  while ($bundle = db_fetch_object($result)) {
    $i++;
    $blocks[$i][subject] = $bundle->title;
58
    $blocks[$i][content] = import_bundle($bundle->attributes, 10);
Dries's avatar
   
Dries committed
59
60
61
62
63
    $blocks[$i][info] = "$bundle->title bundle";
  }
  return $blocks;
}

Dries's avatar
   
Dries committed
64
65
66
67
68
function import_remove($feed) {
  db_query("DELETE FROM item WHERE fid = '$feed[fid]'");
  return "feed '$feed[title]' reset.";
}

Dries's avatar
   
Dries committed
69
function import_update($feed) {
70
71
72
73
74
75
76
77
78
79
80
81

  // open socket:
  $url = parse_url($feed[link]);
  $fp = fsockopen($url[host], ($url[port] ? $url[port] : 80), $errno, $errstr, 15);

  if ($fp) {
    // fetch data:
    fputs($fp, "GET $url[path]?$url[query] HTTP/1.0\nUser-Agent: ". variable_get(site_name, "drupal") ."\nHost: $url[host]\nAccept: */*\n\n");
    while(!feof($fp)) $data .= fgets($fp, 128);

    if (strstr($data, "200 OK")) {

Dries's avatar
   
Dries committed
82
83
84
      eregi("<item([^s].*)</item>", $data, $data);

      // print "<PRE>". htmlentities($data[0]) ."</PRE>";
85
86
87

      foreach (explode("</item>", $data[0]) as $item) {
        $t = eregi("<title>(.*)</title>", $item, $title);
Dries's avatar
   
Dries committed
88
        $l = eregi("<link>(.*)</link>", $item, $link);
89
90
91
92
        $a = eregi("<author>(.*)</author>", $item, $author);
        $d = eregi("<description>(.*)</description>", $item, $description);

        if ($l || $t || $a || $d) {
93
          import_save_item(array(fid => $feed[fid], title => $title[0], link => $link[0], author => $author[0], description => $description[0], attributes => $feed[attributes]));
94
95
96
97
98
99
100
101
102
        }
      }

      db_query("UPDATE feed SET timestamp = '". time() ."' WHERE fid = '". $feed[fid] ."'");
    }
    else {
      watchdog("error", "failed to syndicate from '$feed[title]'");
    }
  }
Dries's avatar
   
Dries committed
103
104

  return "feed '$feed[title]' updated.";
105
106
}

Dries's avatar
   
Dries committed
107
function import_save_item($edit) {
108
  if ($edit[iid] && $edit[title]) {
109
    db_query("UPDATE item SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', author = '". check_input($edit[author]) ."', description = '". check_input($edit[description]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE iid = '$edit[iid]'");
110
111
112
113
114
115
  }
  else if ($edit[iid]) {
    db_query("DELETE FROM item WHERE iid = '". check_input($edit[iid]) ."'");
  }
  else {
    if (!db_fetch_object(db_query("SELECT iid FROM item WHERE link = '". check_input($edit[link]) ."'"))) {
116
      db_query("INSERT INTO item (fid, title, link, author, description, attributes, timestamp) VALUES ('". check_input($edit[fid]) ."', '". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[author]) ."', '". check_input($edit[description]) ."', '". check_input($edit[attributes]) ."', '". time() ."')");
117
118
119
120
    }
  }
}

Dries's avatar
   
Dries committed
121
function import_form_bundle($edit = array()) {
122
123
124
  global $REQUEST_URI;

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the bundle.");
125
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the bundle.");
126
127
128
129
130
131
132
133
134
135
136

  $form .= form_submit("Submit");

  if ($edit[bid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("bid", $edit[bid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
   
Dries committed
137
function import_save_bundle($edit) {
138
  if ($edit[bid] && $edit[title]) {
139
    db_query("UPDATE bundle SET title = '". check_input($edit[title]) ."', attributes = '". check_input($edit[attributes]) ."' WHERE bid = '". check_input($edit[bid]) ."'");
140
141
142
143
144
  }
  else if ($edit[bid]) {
    db_query("DELETE FROM bundle WHERE bid = '". check_input($edit[bid]) ."'");
  }
  else {
145
    db_query("INSERT INTO bundle (title, attributes) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[attributes]) ."')");
146
  }
Dries's avatar
   
Dries committed
147
148

  module_rehash_blocks("import");
149
150
}

Dries's avatar
   
Dries committed
151
function import_form_feed($edit = array()) {
152
153
154
155
156
  global $REQUEST_URI;

  $period = array(900 => format_interval(900), 1800 => format_interval(1800), 3600 => format_interval(3600), 7200 => format_interval(7200), 10800 => format_interval(10800), 21600 => format_interval(21600), 32400 => format_interval(32400), 43200 => format_interval(43200), 64800 => format_interval(64800), 86400 => format_interval(86400), 172800 => format_interval(172800), 259200 => format_interval(259200), 604800 => format_interval(604800), 1209600 => format_interval(1209600), 2419200 => format_interval(2419200));

  $form .= form_textfield("Title", "title", $edit[title], 50, 64, "The name of the feed; typically the name of the website you syndicate content from.");
Dries's avatar
   
Dries committed
157
  $form .= form_textfield("Link", "link", $edit[link], 50, 128, "The fully-qualified URL of the feed.");
158
  $form .= form_textfield("Attributes", "attributes", $edit[attributes], 50, 128, "A comma-seperated list of keywords describing the feed.");
159
160
161
162
163
164
165
166
167
168
169
170
171
  $form .= form_select("Update interval", "refresh", $edit[refresh], $period, "The refresh interval indicating how often you want to update this feed.  Requires crontab.");
  $form .= form_select("Expiration time", "uncache", $edit[uncache], $period, "The time cached items should be kept.  Older items will be automatically discarded.  Requires crontab.");

  $form .= form_submit("Submit");

  if ($edit[fid]) {
    $form .= form_submit(t("Delete"));
    $form .= form_hidden("fid", $edit[fid]);
  }

  return form($REQUEST_URI, $form);
}

Dries's avatar
   
Dries committed
172
function import_save_feed($edit) {
173
  if ($edit[fid] && $edit[title]) {
174
    db_query("UPDATE feed SET title = '". check_input($edit[title]) ."', link = '". check_input($edit[link]) ."', attributes = '". check_input($edit[attributes]) ."', refresh = '". check_input($edit[refresh]) ."', uncache = '". check_input($edit[uncache]) ."' WHERE fid = '". check_input($edit[fid]) ."'");
175
176
177
178
179
180
181
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else if ($edit[fid]) {
    db_query("DELETE FROM feed WHERE fid = '". check_input($edit[fid]) ."'");
    db_query("DELETE FROM item WHERE fid = '". check_input($edit[fid]) ."'");
  }
  else {
182
    db_query("INSERT INTO feed (title, link, attributes, refresh, uncache) VALUES ('". check_input($edit[title]) ."', '". check_input($edit[link]) ."', '". check_input($edit[attributes]) ."', '". check_input($edit[refresh]) ."', '". check_input($edit[uncache]) ."')");
183
184
185
  }
}

Dries's avatar
   
Dries committed
186
function import_save_attributes($edit) {
187
  foreach($edit as $iid => $value) {
188
    db_query("UPDATE item SET attributes = '". check_input($value) ."' WHERE iid = '". check_input($iid) ."'");
189
190
191
192
  }
  return "attributes has been saved";
}

Dries's avatar
   
Dries committed
193
function import_get_feed($fid) {
194
195
196
  return db_fetch_array(db_query("SELECT * FROM feed WHERE fid = '". check_input($fid) ."'"));
}

Dries's avatar
   
Dries committed
197
function import_get_bundle($bid) {
198
199
200
  return db_fetch_array(db_query("SELECT * FROM bundle WHERE bid = '". check_input($bid) ."'"));
}

Dries's avatar
   
Dries committed
201
function import_view_feed() {
202
203
  $result = db_query("SELECT f.*, COUNT(i.iid) AS items FROM feed f LEFT JOIN item i ON f.fid = i.fid GROUP BY f.fid ORDER BY f.title");

Dries's avatar
   
Dries committed
204
  $output .= "<H3>Feed overview</H3>";
205
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
Dries's avatar
   
Dries committed
206
  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>items</TH><TH>last update</TH><TH>next update</TH><TH COLSPAN=\"3\">operations</TH></TR>\n";
207
  while ($feed = db_fetch_object($result)) {
208
    $output .= " <TR><TD>". check_output($feed->title) ."</TD><TD>". check_output($feed->attributes) ."</TD><TD>". format_plural($feed->items, "item", "items") ."</TD><TD>". ($feed->timestamp ? format_interval(time() - $feed->timestamp) ." ago" : "never") ."</TD><TD>". ($feed->timestamp ? format_interval($feed->timestamp + $feed->refresh - time()) ." left" : "never") ."</TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=edit&id=$feed->fid\">edit feed</A></TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=remove&id=$feed->fid\">remove items</A></TD><TD><A HREF=\"admin.php?mod=import&type=feed&op=update&id=$feed->fid\">update items</A></TD></TR>\n";
209
210
211
212
213
  }
  $output .= "</TABLE>\n";

  $result = db_query("SELECT * FROM bundle ORDER BY title");

Dries's avatar
   
Dries committed
214
  $output .= "<H3>Bundle overview</H3>";
215
216
217
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
  $output .= " <TR><TH>title</TH><TH>attributes</TH><TH>operations</TH></TR>\n";
  while ($bundle = db_fetch_object($result)) {
218
    $output .= " <TR><TD>". check_output($bundle->title) ."</TD><TD>". check_output($bundle->attributes) ."</TD><TD><A HREF=\"admin.php?mod=import&type=bundle&op=edit&id=$bundle->bid\">edit bundle</A></TD></TR>\n";
219
220
221
222
223
224
  }
  $output .= "</TABLE>\n";

  return $output;
}

Dries's avatar
   
Dries committed
225
function import_view_item() {
226
227
228
229
230
231
232
233
  global $REQUEST_URI;

  $result = db_query("SELECT i.*, f.title AS feed FROM item i LEFT JOIN feed f ON i.fid = f.fid ORDER BY i.timestamp DESC LIMIT 50");

  $output .= "<FORM ACTION=\"$REQUEST_URI\" METHOD=\"post\">\n";
  $output .= "<TABLE BORDER=\"1\" CELLSPADDING=\"2\" CELLSPACING=\"2\">\n";
  $output .= " <TR><TH>time</TH><TH>feed</TH><TH>item</TH></TR>\n";
  while ($item = db_fetch_object($result)) {
234
    $output .= " <TR><TD VALIGN=\"top\" NOWRAP>". format_date($item->timestamp, "custom", "m/d/y") ."<BR>".format_date($item->timestamp, "custom", "H:i") ."</TD><TD ALIGN=\"center\" VALIGN=\"top\" NOWRAP><A HREF=\"admin.php?mod=import&type=feed&op=edit&id=$item->fid\">". check_output($item->feed) ."</A></TD><TD><A HREF=\"". check_output($item->link) ."\">". check_output($item->title) ."</A>". ($item->description ? "<BR><SMALL><I>". check_output($item->description) ."</I></SMALL>" : "") ."<BR><INPUT TYPE=\"text\" NAME=\"edit[$item->iid]\" VALUE=\"". check_form($item->attributes) ."\" SIZE=\"50\"></TD></TR>\n";
235
236
237
238
239
240
241
242
  }
  $output .= "</TABLE>\n";
  $output .= "<INPUT TYPE=\"submit\" NAME=\"op\" VALUE=\"Save attributes\">\n";
  $output .= "</FORM>\n";

  return $output;
}

Dries's avatar
   
Dries committed
243
function import_admin() {
Dries's avatar
   
Dries committed
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
  global $user, $op, $id, $type, $edit;

  if (user_access($user, "add and edit news feeds")) {

    print "<SMALL><A HREF=\"admin.php?mod=import&type=bundle&op=add\">add new bundle</A> | <A HREF=\"admin.php?mod=import&type=feed&op=add\">add new feed</A> | <A HREF=\"admin.php?mod=import&type=bundle&op=view\">available bundles</A> | <A HREF=\"admin.php?mod=import&type=item&op=view\">available items</A> | <A HREF=\"admin.php?mod=import&op=view\">overview</A> | <A HREF=\"admin.php?mod=import&op=help\">help</A></SMALL><HR>";

    switch($op) {
      case "help":
        print import_help();
        break;
      case "add":
        if ($type == "bundle")
          print import_form_bundle();
        else
          print import_form_feed();
        break;
      case "edit":
        if ($type == "bundle")
          print import_form_bundle(import_get_bundle($id));
        else
          print import_form_feed(import_get_feed($id));
        break;
      case "remove":
        print status(import_remove(import_get_feed($id)));
        print import_view_feed();
        break;
      case "update":
        print status(import_update(import_get_feed($id)));
Dries's avatar
   
Dries committed
272
        print import_view_feed();
Dries's avatar
   
Dries committed
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
        break;
      case "Save attributes":
        print status(import_save_attributes($edit));
        print import_view_item();
        break;
      case "Delete":
        $edit[title] = 0;
        // fall through:
      case "Submit":
        if ($type == "bundle")
          print status(import_save_bundle($edit));
        else
          print status(import_save_feed($edit));
        // fall through:
      default:
        if ($type == "bundle")
          print import_view_bundle();
        else if ($type == "item")
          print import_view_item();
        else
          print import_view_feed();
    }
  }
  else {
    print message_access();
298
299
300
301
  }
}

?>