Monday, April 04, 2011 at 1:06 AM.
river2Suite.readFeed
on readFeed (url) {
<<Changes
<<3/31/11; 4:41:09 PM by DW
<<Add eTag support.
<<http://www.pocketsoap.com/weblog/stories/2002/05/0015.html
<<3/31/11; 1:07:14 PM by DW
<<Read the text of the feed here, pass it to xml.rss.getFeedItems, but only if the text changed.
<<5/17/10; 5:06:47 PM by DW
<<If the feed is not enabled, don't read it. Makes it easy to stop feeds that are updating more than is reasonable.
<<12/24/09; 8:43:34 AM by DW
<<Hook into cloudpipe on new items if the server is enabled.
<<9/15/09; 11:24:46 AM by DW
<<Read all the photos in a newly-subscribed-to photo feed.
<<9/10/09; 11:27:04 AM by DW
<<If the items don't have guids, synthesize them by mashing up title, pubdate and link.
<<9/6/09; 3:46:55 PM by DW
<<When we're booting up for the first time, let all news items through. Don't want an empty news page!
<<9/1/09; 11:24:53 AM by DW
<<If the number of errors exceeds the max, unsub from the feed.
<<8/30/09; 5:25:22 PM by DW
<<Track read errors.
<<6/16/09; 10:50:14 AM by DW
<<Created.
local (adrdata = river2Suite.init (), items, feedinfo, adritem, now = clock.now (), startticks = clock.ticks (), flphotofeed, headers);
local (adrfeed = river2suite.initfeed (url), urllist, timeout = 60 * adrdata^.prefs.ctSecsTimeout, flfeedtextchanged);
if not adrfeed^.prefs.enabled { //5/17/10 by DW
return (false)};
try {
bundle { //set urllist
try {
urllist = string.urlsplit (url)}
else {
urllist = string.urlsplit (url + "/")}};
bundle { //set up headers table
new (tabletype, @headers);
if defined (adrfeed^.data.eTag) {
headers.["If-None-Match"] = adrfeed^.data.eTag}};
local (s = tcp.httpClient (server:urllist [2], path:urllist [3], flMessages:false, ctFollowRedirects:5, timeOutTicks:timeout, adrHdrTable:@headers));
local (code = string.nthField (s, ' ', 2));
s = string.httpResultSplit (s, @headers); //; scratchpad.headers = headers
if defined (headers.eTag) {
adrfeed^.data.eTag = headers.eTag};
bundle { //set flfeedtextchanged
if code == "304" {
flfeedtextchanged = false;
adrfeed^.stats.ct304s++;
adrdata^.stats.ct304s++}
else {
if s == adrfeed^.data.feedtext {
flfeedtextchanged = false}
else {
flfeedtextchanged = true}}};
if flfeedtextchanged {
xml.rss.getFeedItems (url, @items, @adrfeed^.feedinfo);
adrfeed^.data.feedtext = s;
adrfeed^.stats.ctFeedTextChanges++;
adrfeed^.stats.whenLastFeedTextChange = now}}
else {
local (errorstring = "Error reading <a href=\"" + url + "\">feed</a>: " + tryerror);
adrfeed^.stats.ctReadErrors++;
if ++adrfeed^.stats.ctConsecutiveReadErrors >= adrdata^.prefs.maxConsecutiveFeedErrors {
river2Suite.unsubscribeFeed (adrfeed);
log2.add (river2Info.name, "Unsub", errorstring, startticks)}
else {
adrfeed^.stats.whenLastReadError = now;
adrfeed^.stats.lastReadError = tryerror;
log2.add (river2Info.name, "Read", errorstring, startticks)};
return (false)};
bundle { //debugging
if adrdata^.prefs.flDebug {
local (adrfeeds = @system.temp.river2.debug.feeds);
if not defined (adrfeeds^) {
new (tabletype, adrfeeds)};
local (adrfeed = @adrfeeds^.[url]);
if not defined (adrfeed^) {
new (tabletype, adrfeed)};
adrfeed^.items = items}};
adrfeed^.stats.ctReads++;
adrfeed^.stats.whenLastRead = now;
if flfeedtextchanged {
adrfeed^.stats.ctItemsLastRead = sizeof (items);
bundle { //set flphotofeed, 9/15/09 by DW
flphotofeed = false;
if defined (adrfeed^.feedinfo.category) {
flphotofeed = string.lower (adrfeed^.feedinfo.category) == "photos"}};
<<scratchpad.items = items
for adritem in @items {
local (guid);
bundle { //set guid
try {
guid = xml.getvalue (adritem, "guid")}
else {
guid = ""; //synthesize the guid -- 9/10/09 by DW
try {guid = guid + xml.getvalue (adritem, "pubDate")};
try {guid = guid + xml.getvalue (adritem, "link")};
try {guid = guid + xml.getvalue (adritem, "title")};
if sizeof (guid) > 0 {
guid = string.hashmd5 (guid)}}};
if sizeof (guid) > 0 {
local (adrhistory = @adrfeed^.history.[guid]);
if not defined (adrhistory^) {
if (adrfeed^.stats.ctReads > 1) or adrdata^.stats.flInstalling or flphotofeed {
<<first time we read a feed none of the items are new, unless we're installing or it's a photo feed
river2suite.newItem (adritem, adrfeed);
if adrdata^.prefs.cloudPipeServer.enabled { //12/24/09 by DW
river2Suite.cloudPipe.serverNewItem (url, adritem)}};
adrhistory^ = now}}}};
return (true)};
bundle { //test code
readFeed ("http://scripting.com/rss.xml")}
<<readFeed ("http://static.flickrfan.org/afp/rss.xml")
This listing is for code that runs in the OPML Editor environment. I created these listings because I wanted the search engines to index it, so that when I want to look up something in my codebase I don't have to use the much slower search functionality in my object database. Dave Winer.