on compileService (adrservice, flSaveData=false, adrStoryArrivedCallback=nil) { «Changes «3/17/02; 4:16:05 PM by DW «In looking for the name of the top-level element of an unknown type feed, don't stop on DOCTYPE. «http://radio.userland.com/discuss/msgReader$12010 «3/15/02; 2:20:50 PM by DW «Add a driver architecture to the aggregator so that new formats can be added without changing the core of the aggregator. «http://radio.userland.com/aggregatorDriverArchitecture «12/12/01; 5:21:54 PM by JES «Pass in true for the value of flSkipMalformedEntities when calling xml.entitydecode. When adding an item to the history if sizeOf (title) > 0, don't add the item. (See locally declared addToHistory.) «2/22/01; 9:29:50 PM by JES «If this is a Mac, convert to Mac text when decoding strings in the service XML. «2/16/01; 12:21:35 PM by PBS «Some elements weren't being decoded; now they all are. «1/12/01; 6:06:29 AM by DW «An important fix. In 0.92 the sub-element of is optional, but we didn't treat it that way, the compilation would fail if any item didn't have a sub-element. «1/11/01; 6:32:59 PM by DW «Add support for s. «1/5/01; 5:20:16 PM by DW «Add support for publish-and-subscribe notification. «1/1/01; 11:54:32 AM by DW «Hit the limit on the size of addresses that can be processed by xml.getAddressList. «I could re-code all the loops over the items to work around this limit or I could do something more clever, work with a local copy of the xmlstruct table. This would probably be faster too. «In fact in earlier versions of this script it worked this way, explaining why I'm hitting the problem on some channels now. Interesting. «Thursday, December 28, 2000 at 7:26:03 AM by DW «Added adrStoryArrivedCallback, it's called when a new story has arrived. You can store it in a database, or whatever else you might want to do. «Wednesday, December 27, 2000 at 2:27:50 PM by DW «Added flSaveData boolean, if true, we remember the data behind the title in the item table. This is needed for processing RSS channels in Radio's outliner, we need the text and the link separated, unmashed together. «Wednesday, December 27, 2000 at 12:17:26 PM by DW «Moved into Radio.root and Frontier.root so it can be accessed by Manila and the new nodetype code in 7.0. «Sunday, December 17, 2000 at 10:49:04 AM by DW «Radically simplified for Radio use. Lots of ancient stuff swept away. «6/19/99; 9:34:46 PM by DW «Created. on decode (s) { return (xml.rss.decodeString (s))}; xml.rss.init (); //set up user.xml.rss local (adrcompilation = @adrservice^.compilation); if not defined (adrcompilation^) { new (tabletype, adrcompilation)}; local (format = "unknown"); local (xstruct, adrxstruct = @xstruct); bundle { //comple the xml text, set the format xml.compile (string (adrservice^.xmltext), adrxstruct); adrservice^.xmlstruct = adrxstruct^; try { xml.getAddress (adrxstruct, "RDF"); format = "RSS1"} else { try { xml.getAddress (adrxstruct, "scriptingNews"); format = "scriptingNews"} else { try { xml.getAddress (adrxstruct, "rss"); format = "RSS"}}}}; local (adrhistory = @adrcompilation^.itemHistory); bundle { //initialize itemHistory table local (i); if not defined (adrhistory^) { new (tabletype, adrhistory)}; for i = 1 to sizeof (adrhistory^) { adrhistory^ [i] = false}}; on addToHistory (adritem, categorylist={}) { local (title = adritem^.title); if sizeOf (title) > 0 { local (adrinhistory = @adrhistory^.[title]); if not defined (adrinhistory^) { //it's a new story if adrStoryArrivedCallback != nil { adrStoryArrivedCallback^ (adrservice, adritem)}}; adrinhistory^ = true}}; //it's current case format { "RSS1" { local (adrrdf = xml.getAddress (adrxstruct, "RDF")); bundle { //set channeltitle, channellink, channeldescription local (adrchannel = decode (xml.getAddress (adrrdf, "channel"))); adrcompilation^.channeltitle = decode (xml.getValue (adrchannel, "title")); adrcompilation^.channellink = decode (xml.getValue (adrchannel, "link")); adrcompilation^.channeldescription = decode (xml.getValue (adrchannel, "description"))}; bundle { //set imagetitle, imageurl, imagelink try { //the image is not required local (adrimage = xml.getAddress (adrchannel, "image")); adrcompilation^.imagetitle = decode (xml.getValue (adrimage, "title")); adrcompilation^.imageurl = decode (xml.getValue (adrimage, "url")); adrcompilation^.imagelink = decode (xml.getValue (adrimage, "link"))}}; bundle { //set textinput stuff try { //textinput is not required local (adrtextinput = xml.getAddress (adrrdf, "textinput")); adrcompilation^.textInputTitle = decode (xml.getValue (adrtextinput, "title")); adrcompilation^.textInputDescription = decode (xml.getValue (adrtextinput, "description")); adrcompilation^.textInputName = decode (xml.getValue (adrtextinput, "name")); adrcompilation^.textInputLink = decode (xml.getValue (adrtextinput, "link"))}}; local (itemlist = xml.getAddressList (adrrdf, "item"), item, ct = 1, i); new (tabletype, @adrcompilation^.items); for i = sizeof (itemlist) downto 1 { //each item is the address of an item item = itemlist [i]; adritem = @adrcompilation^.items.[string.padwithzeros (ct++, 5)]; new (tabletype, adritem); local (link = decode (xml.getValue (item, "link"))); local (title = decode (xml.getValue (item, "title"))); local (description = ""); try {description = decode (xml.getValue (item, "description"))}; adritem^.title = "" + title + ""; if sizeof (description) > 0 { adritem^.title = adritem^.title + ". " + description}; if flSaveData { local (adrdata = @adritem^.data); new (tabletype, adrdata); adrdata^.link = link; adrdata^.title = title; adrdata^.description = description}; addToHistory (adritem); }};«adritem^.link = xml.getValue (item, "link") "RSS" { //version 0.91 or 0.92 local (adrRss = xml.getAddress (adrxstruct, "rss")); local (adrchannel = xml.getAddress (adrRss, "channel")); local (version = xml.getAttribute (adrRss, "version")^); local (flVersion91 = date.versionlessthan (version, "0.92d1")); adrcompilation^.channeltitle = decode (xml.getValue (adrchannel, "title")); adrcompilation^.channellink = decode (xml.getValue (adrchannel, "link")); adrcompilation^.channeldescription = decode (xml.getValue (adrchannel, "description")); bundle { //get language, it's optional in 0.92 and greater if flVersion91 { adrcompilation^.channellanguage = decode (xml.getValue (adrchannel, "language"))} else { try { adrcompilation^.channellanguage = decode (xml.getValue (adrchannel, "language"))}}}; bundle { //get the cloud element, new in 0.92, optional try { local (cloudinfo); new (tabletype, @cloudinfo); local (adrcloud = xml.getaddress (adrchannel, "cloud")); cloudinfo.domain = decode (xml.getAttribute (adrcloud, "domain")^); cloudinfo.path = decode (xml.getAttribute (adrcloud, "path")^); cloudinfo.port = decode (xml.getAttribute (adrcloud, "port")^); cloudinfo.protocol = decode (xml.getAttribute (adrcloud, "protocol")^); cloudinfo.registerProcedure = decode (xml.getAttribute (adrcloud, "registerProcedure")^); adrcompilation^.cloud = cloudinfo}}; bundle { //set imagetitle, imageurl, imagelink try { //the image is not required local (adrimage = xml.getAddress (adrchannel, "image")); adrcompilation^.imagetitle = decode (xml.getValue (adrimage, "title")); adrcompilation^.imageurl = decode (xml.getValue (adrimage, "url")); adrcompilation^.imagelink = decode (xml.getValue (adrimage, "link")); adrcompilation^.imagewidth = decode (xml.getValue (adrimage, "width")); adrcompilation^.imageheight = decode (xml.getValue (adrimage, "height")); adrcompilation^.imagedescription = decode (xml.getValue (adrimage, "description"))}}; bundle { //set textinput stuff try { //textinput is not required local (adrtextinput = xml.getAddress (adrRss, "textinput")); adrcompilation^.textInputTitle = decode (xml.getValue (adrtextinput, "title")); adrcompilation^.textInputDescription = decode (xml.getValue (adrtextinput, "description")); adrcompilation^.textInputName = decode (xml.getValue (adrtextinput, "name")); adrcompilation^.textInputLink = decode (xml.getValue (adrtextinput, "link"))}}; bundle { //get the items local (itemlist = xml.getAddressList (adrchannel, "item"), item, ct = 1, i); new (tabletype, @adrcompilation^.items); for i = sizeof (itemlist) downto 1 { //each item is the address of an item item = itemlist [i]; adritem = @adrcompilation^.items.[string.padwithzeros (ct++, 5)]; new (tabletype, adritem); local (link = "", title = "", description = "", s = ""); try {link = decode (xml.getValue (item, "link"))}; try {title = decode (xml.getValue (item, "title"))}; try {description = decode (xml.getValue (item, "description"))}; if (sizeof (title) > 0) and (sizeof (link) > 0) { s = "" + title + ""}; if sizeof (description) > 0 { if sizeof (s) > 0 { s = s + ". " + description} else { s = description}}; adritem^.title = s; if flSaveData { local (adrdata = @adritem^.data); new (tabletype, adrdata); adrdata^.link = link; adrdata^.title = title; adrdata^.description = description}; bundle { //get enclosure, 1/11/01; 6:32:52 PM by DW try { local (adrxenclosure = xml.getaddress (item, "enclosure")); local (adrenclosure = @adritem^.enclosure); new (tabletype, adrenclosure); adrenclosure^.url = decode (xml.getattribute (adrxenclosure, "url")^); adrenclosure^.length = decode (xml.getattribute (adrxenclosure, "length")^); adrenclosure^.type = decode (xml.getattribute (adrxenclosure, "type")^)}}; local (catlist = {}); bundle { //9/30/99; 7:30:12 PM by DW, get the categories, if there are any local (xcatlist = xml.getAddressList (item, "category"), cat); for cat in xcatlist { catlist = catlist + {decode (cat^)}}}; addToHistory (adritem, catlist); bundle { //get source, it's optional in 0.92 and greater if not flVersion91 { try { local (adrsource = xml.getAddress (item, "source")); local (url = decode (xml.getAttribute (adrsource, "url")^)); adritem^.sourceChannelTitle = decode (adrsource^.["/pcdata"]); adritem^.sourceChannelUrl = url}}}}}}; "scriptingNews" { local (adrScriptingNews = xml.getAddress (adrxstruct, "scriptingNews")); local (adrheader = xml.getAddress (adrScriptingNews, "header")); local (adrversion = xml.getAddress (adrheader, "scriptingNewsVersion")); if not date.versionLessThan (adrversion^, "2.0b1") { adrcompilation^.channeltitle = decode (xml.getValue (adrheader, "channelTitle")); adrcompilation^.channellink = decode (xml.getValue (adrheader, "channelLink")); adrcompilation^.channeldescription = decode (xml.getValue (adrheader, "channelDescription"))} else { adrcompilation^.channeltitle = ""; adrcompilation^.channellink = ""; adrcompilation^.channeldescription = ""}; bundle { //set imagetitle, imageurl, imagelink try { //the image is not required adrcompilation^.imagetitle = decode (xml.getValue (adrheader, "imageTitle")); adrcompilation^.imageurl = decode (xml.getValue (adrheader, "imageUrl")); adrcompilation^.imagelink = decode (xml.getValue (adrheader, "imageLink"))}}; bundle { //get skipHours try { //it's optional local (adrskip = xml.getAddress (adrheader, "skipHours")); local (skiplist = xml.getAddressList (adrskip, "hour"), item); adrcompilation^.skipHours = {}; for item in skiplist { //each item is the address of an item adrcompilation^.skipHours = adrcompilation^.skipHours + {decode (item^)}}}}; bundle { //get skipDays try { //it's optional local (adrskip = xml.getAddress (adrheader, "skipDays")); local (skiplist = xml.getAddressList (adrskip, "day"), item); adrcompilation^.skipDays = {}; for item in skiplist { //each item is the address of an item adrcompilation^.skipDays = adrcompilation^.skipDays + {decode (item^)}}}}; bundle { //get various optional things from the header try {adrcompilation^.managingEditor = decode (xml.getValue (adrheader, "managingEditor"))}; try {adrcompilation^.webmaster = decode (xml.getValue (adrheader, "webmaster"))}; try {adrcompilation^.language = decode (xml.getValue (adrheader, "language"))}; try {adrcompilation^.imageHeight = decode (xml.getValue (adrheader, "imageHeight"))}; try {adrcompilation^.imageWidth = decode (xml.getValue (adrheader, "imageWidth"))}; try {adrcompilation^.imageCaption = decode (xml.getValue (adrheader, "imageCaption"))}}; bundle { //load the items local (itemlist = xml.getAddressList (adrScriptingNews, "item")); local (item, ct = 1, adritem, itemtext, i); new (tabletype, @adrcompilation^.items); for i = sizeof (itemlist) downto 1 { //each item is the address of an item item = itemlist [i]; adritem = @adrcompilation^.items.[string.padwithzeros (ct++, 5)]; new (tabletype, adritem); itemtext = decode (xml.getValue (item, "text")); «itemtext = string.replaceall (itemtext, "<", "<") «itemtext = string.replaceall (itemtext, ">", ">") itemtext = decode (itemtext); local (linklist = xml.getAddressList (item, "link"), link); for link in linklist { local (url = decode (xml.getValue (link, "url"))); local (linetext = decode (xml.getValue (link, "linetext"))); local (s = "" + linetext + ""); itemtext = string.replace (itemtext, linetext, s)}; adritem^.title = itemtext; local (catlist = {}); bundle { //9/30/99; 8:48:02 PM by DW, get the categories, if there are any local (xcatlist = xml.getAddressList (item, "category"), cat); for cat in xcatlist { catlist = catlist + {decode (cat^)}}}; addToHistory (adritem, catlist)}}}; "unknown" { //see if a driver can handle this format «How it works -- 3/15/02; 2:35:19 PM by DW «Drivers are located either in user.xml.rss.formatDrivers or xml.rss.formatDrivers; the former for user-supplied drivers, the latter for system drivers. All new formats supported by UserLand will come in the form of drivers. «First we locate the driver, starting in the user table, then looking in the system table. If we don't locate one, do nothing (as before). If we do, we call the driver script, passing it enough parameters so it can do exactly what one of the internal format handlers does. local (adr, drivername = "", name); for adr in adrxstruct { name = xml.convertToDisplayName (nameof (adr^)); if (not (name beginswith "?")) and (not (name beginswith "/")) { drivername = name; break}}; if drivername != "" { local (adrdriver = @user.xml.rss.formatDrivers.[drivername]); if not defined (adrdriver^) { adrdriver = @xml.rss.formatDrivers.[drivername]}; if defined (adrdriver^) { adrdriver^.compile (adrservice, flSaveData, adrStoryArrivedCallback); format = drivername}}}}; adrcompilation^.format = format; bundle { //clean up itemHistory table local (i); for i = sizeof (adrhistory^) downto 1 { if not adrhistory^ [i] { delete (@adrhistory^ [i])}}}}; «bundle //test code «local (url = "http://rcm.amazon.com/e/cm?t=naviseek&l=st1&search=programming&mode=books&p=102&o=1&f=xml") «local (adrservice = xml.rss.initService (url, @aggregatorData.services)) «new (tabletype, @adrservice^.compilation) «new (tabletype, @adrservice^.compilation.itemHistory) «adrservice^.xmltext = tcp.httpReadUrl (url) «xml.compile (adrservice^.xmltext, @adrservice^.xmlstruct) «compileService (adrservice, true, @xml.aggregator.storyArrivedCallback) «bundle //more test code «new (tableType, @temp.profileData) «script.startProfile (true) «compileService (@aggregatorData.services.["http://scriptingnews.userland.com/xml/scriptingNews2.xml"], true, @xml.aggregator.storyArrivedCallback) «script.stopProfile (@temp.profileData) «edit (@temp.profileData)