Monday, November 08, 2010 at 12:07 AM.

---

system.verbs.builtins.xml.aggregator.readAllServices

on readAllServices (maxStories=infinity, adrlogcallback=nil, adrdata=nil) {
	<<Changes:
		<<12/27/01; 1:31:55 PM by PBS
			<<Pass adrdata to xml.aggregator.unsubscribeErrantServices, xml.aggregator.clearCache.
			<<When calling preScan and postScan callbacks, call scripts in adrdata^.callbacks, not in aggregatorData.callbacks.
		<<12/4/01; 7:22:27 PM by DW
			<<Call scripts in aggregatorData.callbacks.preScan before doing scan, in aggregatorData.callbacks.postScan after finishing scan.
		<<12/3/01; 6:06:45 PM by DW
			<<Call xml.aggregator.unsubscribeErrantServices before scanning.
		<<11/30/01; 2:18:19 PM by DW
			<<Set a boolean, adrdata^.stats.lastRun.flScanRunningNow, when you start and finish.
		<<11/7/01; 7:04:22 PM by DW
			<<Change system.temp.aggregator to adrdata^.stats.lastRun.
		<<10/30/01; 7:35:53 AM by DW
			<<Call radio.aggregator.preCacheNewsPage after scan if running in Radio.
		<<10/26/01; 7:16:09 PM by DW
			<<Change final status message to include the number of stories.
		<<8/14/01; 7:32:07 PM by JES
			<<New optional parameter, adrdata, is the address for aggregator data storage. Default is to get the address by calling xml.aggregator.init.
		<<2/23/01; 2:11:17 PM by PBS
			<<New optional parameter, maxStories. It's not a hard maximum, the number of new stories may go a little over. We don't stop compiling all the stories in a service just because we're going a little over the limit. This is used at first time root run, so we don't read 250 stories.
		<<1/6/01; 8:43:58 PM by DW
			<<Logging code is now controlled by a preference.
		<<1/2/01; 5:02:11 PM by DW
			<<Save services list after scan is complete.
		<<12/30/00; 7:58:28 PM by DW
			<<Added logging code.
		<<Sunday, December 17, 2000 at 10:27:43 AM by DW
			<<Created
	
	local (startticks = clock.ticks ());
	
	if adrdata == nil {
		adrdata = xml.aggregator.init ()}
	else {
		xml.aggregator.init ()};
	local (adrservices = @adrdata^.services);
	
	xml.aggregator.unsubscribeErrantServices (adrdata); //unsubscribe from channels with too many errors //PBS 12/27/01: pass adrdata
	
	on statusmsg (s) {
		adrdata^.stats.lastRun.status = s;
		msg (clock.now () + "; " + s)};
	bundle { //init stats
		adrdata^.stats.lastRun.ctThreads = 0;
		adrdata^.stats.lastRun.lastChannelRead = "";
		adrdata^.stats.lastRun.ctNewStories = 0;
		adrdata^.stats.lastRun.flScanRunningNow = true};
	
	on doCallbacks (adrtable) {
		local (adrscript);
		for adrscript in adrtable {
			try {
				while typeof (adrscript^) == addresstype {
					adrscript = adrscript^};
				adrscript^ ()}}};
	doCallbacks (@adrdata^.callbacks.preScan);
	
	local (randomtable); //read the channels in random order
	bundle { //initialize random table
		local (adritem);
		new (tabletype, @randomtable);
		for adritem in adrservices {
			loop { //add service to random table
				adrrandom = @randomtable.[string (random (0, 32767))];
				if not defined (adrrandom^) {
					adrrandom^ = nameof (adritem^);
					break}}}};
	local (cttotalthreads = 0);
	bundle { //main loop
		local (adrrandom, adrservice);
		for adrrandom in @randomtable {
			local (url = adrrandom^);
			adrservice = @adrservices^.[url];
			while adrdata^.stats.lastRun.ctThreads >= adrdata^.prefs.maxThreads { //wait for threadcount to drop below the max
				thread.sleepFor (1)};
			adrdata^.stats.lastRun.ctThreads++;
			thread.callScript ("xml.aggregator.readService",  {url, adrservices});
			cttotalthreads++;
			statusmsg ("Scanning. " + adrdata^.stats.lastRun.ctThreads + " current threads; " + cttotalthreads + " total threads; " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
			if adrdata^.stats.lastRun.ctNewStories >= maxStories {
				break}};
		while adrdata^.stats.lastRun.ctThreads > 0 {
			statusmsg ("Waiting for threads to finish: " + adrdata^.stats.lastRun.ctThreads + " threads.");
			thread.sleepfor (1)}};
	bundle { //add an item to the daily log
		if adrdata^.prefs.flLogHourlyScan {
			if adrlogcallback != nil {
				local (htmltext = cttotalthreads + " channels read, " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
				adrlogcallback^ ("Hourly scan", htmltext, startticks)}}};
	xml.aggregator.clearCache (adrdata); //PBS 12/27/01: pass adrdata
	bundle { //pre-cache the News page, if running in Radio
		if system.environment.isRadio {
			radio.aggregator.preCacheNewsPage ()}};
	filemenu.savemyroot (adrservices);
	statusmsg ("Last scan complete at " + date.timestring () + "; " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
	adrdata^.stats.lastRun.whenComplete = clock.now ();
	adrdata^.stats.lastRun.flScanRunningNow = false;
	doCallbacks (@adrdata^.callbacks.postScan);
	msg ("")}
<<bundle //test code
	<<readAllServices (adrlogcallback:@radio.log.add)

---

This listing is for code that runs in the OPML Editor environment. I created these listings because I wanted the search engines to index it, so that when I want to look up something in my codebase I don't have to use the much slower search functionality in my object database. Dave Winer.