Monday, April 04, 2011 at 1:06 AM.

---

river2Suite.readFeed

on readFeed (url) {
	<<Changes
		<<3/31/11; 4:41:09 PM by DW
			<<Add eTag support.
				<<http://www.pocketsoap.com/weblog/stories/2002/05/0015.html
		<<3/31/11; 1:07:14 PM by DW
			<<Read the text of the feed here, pass it to xml.rss.getFeedItems, but only if the text changed.
		<<5/17/10; 5:06:47 PM by DW
			<<If the feed is not enabled, don't read it. Makes it easy to stop feeds that are updating more than is reasonable.
		<<12/24/09; 8:43:34 AM by DW
			<<Hook into cloudpipe on new items if the server is enabled.
		<<9/15/09; 11:24:46 AM by DW
			<<Read all the photos in a newly-subscribed-to photo feed.
		<<9/10/09; 11:27:04 AM by DW
			<<If the items don't have guids, synthesize them by mashing up title, pubdate and link.
		<<9/6/09; 3:46:55 PM by DW
			<<When we're booting up for the first time, let all news items through. Don't want an empty news page!
		<<9/1/09; 11:24:53 AM by DW
			<<If the number of errors exceeds the max, unsub from the feed.
		<<8/30/09; 5:25:22 PM by DW
			<<Track read errors.
		<<6/16/09; 10:50:14 AM by DW
			<<Created. 
	local (adrdata = river2Suite.init (), items, feedinfo, adritem, now = clock.now (), startticks = clock.ticks (), flphotofeed, headers);
	local (adrfeed = river2suite.initfeed (url), urllist, timeout = 60 * adrdata^.prefs.ctSecsTimeout, flfeedtextchanged);
	if not adrfeed^.prefs.enabled { //5/17/10 by DW
		return (false)};
	try {
		bundle { //set urllist
			try {
				urllist = string.urlsplit (url)}
			else {
				urllist = string.urlsplit (url + "/")}};
		bundle { //set up headers table
			new (tabletype, @headers);
			if defined (adrfeed^.data.eTag) {
				headers.["If-None-Match"] = adrfeed^.data.eTag}};
		local (s = tcp.httpClient (server:urllist [2], path:urllist [3], flMessages:false, ctFollowRedirects:5, timeOutTicks:timeout, adrHdrTable:@headers));
		local (code = string.nthField (s, ' ', 2));
		s = string.httpResultSplit (s, @headers); //; scratchpad.headers = headers
		if defined (headers.eTag) {
			adrfeed^.data.eTag = headers.eTag};
		bundle { //set flfeedtextchanged
			if code == "304" {
				flfeedtextchanged = false;
				adrfeed^.stats.ct304s++;
				adrdata^.stats.ct304s++}
			else {
				if s == adrfeed^.data.feedtext {
					flfeedtextchanged = false}
				else {
					flfeedtextchanged = true}}};
		if flfeedtextchanged {
			xml.rss.getFeedItems (url, @items, @adrfeed^.feedinfo);
			adrfeed^.data.feedtext = s;
			adrfeed^.stats.ctFeedTextChanges++;
			adrfeed^.stats.whenLastFeedTextChange = now}}
	else {
		local (errorstring = "Error reading <a href=\"" + url + "\">feed</a>: " + tryerror);
		adrfeed^.stats.ctReadErrors++;
		if ++adrfeed^.stats.ctConsecutiveReadErrors >= adrdata^.prefs.maxConsecutiveFeedErrors {
			river2Suite.unsubscribeFeed (adrfeed);
			log2.add (river2Info.name, "Unsub", errorstring, startticks)}
		else {
			adrfeed^.stats.whenLastReadError = now;
			adrfeed^.stats.lastReadError = tryerror;
			log2.add (river2Info.name, "Read", errorstring, startticks)};
		return (false)};
	bundle { //debugging
		if adrdata^.prefs.flDebug {
			local (adrfeeds = @system.temp.river2.debug.feeds);
			if not defined (adrfeeds^) {
				new (tabletype, adrfeeds)};
			local (adrfeed = @adrfeeds^.[url]);
			if not defined (adrfeed^) {
				new (tabletype, adrfeed)};
			adrfeed^.items = items}};
	adrfeed^.stats.ctReads++;
	adrfeed^.stats.whenLastRead = now;
	if flfeedtextchanged {
		adrfeed^.stats.ctItemsLastRead = sizeof (items);
		bundle { //set flphotofeed, 9/15/09 by DW
			flphotofeed = false;
			if defined (adrfeed^.feedinfo.category) {
				flphotofeed = string.lower (adrfeed^.feedinfo.category) == "photos"}};
		<<scratchpad.items = items
		for adritem in @items {
			local (guid);
			bundle { //set guid
				try {
					guid = xml.getvalue (adritem, "guid")}
				else {
					guid = ""; //synthesize the guid -- 9/10/09 by DW
					try {guid = guid + xml.getvalue (adritem, "pubDate")};
					try {guid = guid + xml.getvalue (adritem, "link")};
					try {guid = guid + xml.getvalue (adritem, "title")};
					if sizeof (guid) > 0 {
						guid = string.hashmd5 (guid)}}};
			if sizeof (guid) > 0 {
				local (adrhistory = @adrfeed^.history.[guid]);
				if not defined (adrhistory^) {
					if (adrfeed^.stats.ctReads > 1) or adrdata^.stats.flInstalling or flphotofeed {
						<<first time we read a feed none of the items are new, unless we're installing or it's a photo feed
						river2suite.newItem (adritem, adrfeed);
						if adrdata^.prefs.cloudPipeServer.enabled { //12/24/09 by DW
							river2Suite.cloudPipe.serverNewItem (url, adritem)}};
					adrhistory^ = now}}}};
	return (true)};
bundle { //test code
	readFeed ("http://scripting.com/rss.xml")}
	<<readFeed ("http://static.flickrfan.org/afp/rss.xml")

---

This listing is for code that runs in the OPML Editor environment. I created these listings because I wanted the search engines to index it, so that when I want to look up something in my codebase I don't have to use the much slower search functionality in my object database. Dave Winer.