Monday, November 08, 2010 at 12:07 AM.
system.verbs.builtins.xml.aggregator.readAllServices
on readAllServices (maxStories=infinity, adrlogcallback=nil, adrdata=nil) {
<<Changes:
<<12/27/01; 1:31:55 PM by PBS
<<Pass adrdata to xml.aggregator.unsubscribeErrantServices, xml.aggregator.clearCache.
<<When calling preScan and postScan callbacks, call scripts in adrdata^.callbacks, not in aggregatorData.callbacks.
<<12/4/01; 7:22:27 PM by DW
<<Call scripts in aggregatorData.callbacks.preScan before doing scan, in aggregatorData.callbacks.postScan after finishing scan.
<<12/3/01; 6:06:45 PM by DW
<<Call xml.aggregator.unsubscribeErrantServices before scanning.
<<11/30/01; 2:18:19 PM by DW
<<Set a boolean, adrdata^.stats.lastRun.flScanRunningNow, when you start and finish.
<<11/7/01; 7:04:22 PM by DW
<<Change system.temp.aggregator to adrdata^.stats.lastRun.
<<10/30/01; 7:35:53 AM by DW
<<Call radio.aggregator.preCacheNewsPage after scan if running in Radio.
<<10/26/01; 7:16:09 PM by DW
<<Change final status message to include the number of stories.
<<8/14/01; 7:32:07 PM by JES
<<New optional parameter, adrdata, is the address for aggregator data storage. Default is to get the address by calling xml.aggregator.init.
<<2/23/01; 2:11:17 PM by PBS
<<New optional parameter, maxStories. It's not a hard maximum, the number of new stories may go a little over. We don't stop compiling all the stories in a service just because we're going a little over the limit. This is used at first time root run, so we don't read 250 stories.
<<1/6/01; 8:43:58 PM by DW
<<Logging code is now controlled by a preference.
<<1/2/01; 5:02:11 PM by DW
<<Save services list after scan is complete.
<<12/30/00; 7:58:28 PM by DW
<<Added logging code.
<<Sunday, December 17, 2000 at 10:27:43 AM by DW
<<Created
local (startticks = clock.ticks ());
if adrdata == nil {
adrdata = xml.aggregator.init ()}
else {
xml.aggregator.init ()};
local (adrservices = @adrdata^.services);
xml.aggregator.unsubscribeErrantServices (adrdata); //unsubscribe from channels with too many errors //PBS 12/27/01: pass adrdata
on statusmsg (s) {
adrdata^.stats.lastRun.status = s;
msg (clock.now () + "; " + s)};
bundle { //init stats
adrdata^.stats.lastRun.ctThreads = 0;
adrdata^.stats.lastRun.lastChannelRead = "";
adrdata^.stats.lastRun.ctNewStories = 0;
adrdata^.stats.lastRun.flScanRunningNow = true};
on doCallbacks (adrtable) {
local (adrscript);
for adrscript in adrtable {
try {
while typeof (adrscript^) == addresstype {
adrscript = adrscript^};
adrscript^ ()}}};
doCallbacks (@adrdata^.callbacks.preScan);
local (randomtable); //read the channels in random order
bundle { //initialize random table
local (adritem);
new (tabletype, @randomtable);
for adritem in adrservices {
loop { //add service to random table
adrrandom = @randomtable.[string (random (0, 32767))];
if not defined (adrrandom^) {
adrrandom^ = nameof (adritem^);
break}}}};
local (cttotalthreads = 0);
bundle { //main loop
local (adrrandom, adrservice);
for adrrandom in @randomtable {
local (url = adrrandom^);
adrservice = @adrservices^.[url];
while adrdata^.stats.lastRun.ctThreads >= adrdata^.prefs.maxThreads { //wait for threadcount to drop below the max
thread.sleepFor (1)};
adrdata^.stats.lastRun.ctThreads++;
thread.callScript ("xml.aggregator.readService", {url, adrservices});
cttotalthreads++;
statusmsg ("Scanning. " + adrdata^.stats.lastRun.ctThreads + " current threads; " + cttotalthreads + " total threads; " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
if adrdata^.stats.lastRun.ctNewStories >= maxStories {
break}};
while adrdata^.stats.lastRun.ctThreads > 0 {
statusmsg ("Waiting for threads to finish: " + adrdata^.stats.lastRun.ctThreads + " threads.");
thread.sleepfor (1)}};
bundle { //add an item to the daily log
if adrdata^.prefs.flLogHourlyScan {
if adrlogcallback != nil {
local (htmltext = cttotalthreads + " channels read, " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
adrlogcallback^ ("Hourly scan", htmltext, startticks)}}};
xml.aggregator.clearCache (adrdata); //PBS 12/27/01: pass adrdata
bundle { //pre-cache the News page, if running in Radio
if system.environment.isRadio {
radio.aggregator.preCacheNewsPage ()}};
filemenu.savemyroot (adrservices);
statusmsg ("Last scan complete at " + date.timestring () + "; " + adrdata^.stats.lastRun.ctNewStories + " new stories.");
adrdata^.stats.lastRun.whenComplete = clock.now ();
adrdata^.stats.lastRun.flScanRunningNow = false;
doCallbacks (@adrdata^.callbacks.postScan);
msg ("")}
<<bundle //test code
<<readAllServices (adrlogcallback:@radio.log.add)
This listing is for code that runs in the OPML Editor environment. I created these listings because I wanted the search engines to index it, so that when I want to look up something in my codebase I don't have to use the much slower search functionality in my object database. Dave Winer.