* * Ex: <"html"><"head"><"title">Web Raveler Rules<"/title"><"/head"><"body">HI<"img * src='http://www.google.com/logo.gif' size=50"><"/body"><"/html"> * * becomes the following vector: * * <<"html">,<"head">,<"title">,"Web Raveler Rules",<"/title">, * <"/head">,<"body">,"HI",<"img",<"src","'http://www.google.com/logo.gif'">, * <"size","50">>, * <"/body">,<"/html">> * ** * HTML comments are stored in standard tag vectors, but with the second element being * a String, instead of a Vector (differentiated by catching a ClassCastException). Attributes * start with a name string, then a value stringBuffer, then a reference to the parent tag. * * @author The Web Raveler Development Squad * @version 1.0 */ public class HTMLPage implements Serializable { /* variables */ /** a vector of vectors and strings (tags and text, respectively) */ private Vector page; /** hashes references to attribute vectors, keyed by name strings */ private Hashtable attributes; /** hashes references to tag vectors, keyed by name strings */ private Hashtable tags; /** references to the text (StringBuffers) in the page */ private Vector textBlocks; private String docRoot; private String serverRoot; /** storage for additions to the top of the page */ private Vector topExtra; /** storage for additions to the head of the page */ private Vector headExtra; /** storage for additions to the bottom of the page */ private Vector bottomExtra; /** * Construct an empty HTMLPage. */ public HTMLPage(){ page = new Vector(); attributes = new Hashtable(); tags = new Hashtable(); textBlocks = new Vector(); }//HTMLPage /** * Construct an HTMLPage from the given string. * * @param pageHTML a String containing the raw HTML we're working with */ public HTMLPage(String pageHTML) { page = new Vector(); attributes = new Hashtable(); tags = new Hashtable(); textBlocks = new Vector(); int low = 0; /* first index of the current tag */ int high = 0; /* last index of the current tag */ int length = pageHTML.length(); /* length of pageHTML */ //System.out.println("page length = '"+length+"'"); /* at each iteration, everything in pageHTML up to, but not including, * low has been parsed. */ while ( low < length ) { /* low starts a tag */ if ( pageHTML.charAt(low) == '<' ) { high = getTag(pageHTML,low); low = high; }//if /* low starts a text block */ else { low = getTextBlock(pageHTML,low); high = low; }//else }//while ( low < length() ) }//HTMLPage(String pageHTML) /** * This method returns the index one past the end of the text block * in the HTML source that starts at index low. * * @param pageHTML a String containing the source HTML * @param low an int, the starting index for our search * @param an int that's the index of the end of the text block started at low */ private int getTextBlock(String pageHTML, int low) { //System.out.println("Getting text block..."); StringBuffer textBlock; int i = pageHTML.indexOf("<",low); if ( i == -1 ) i = pageHTML.length(); /* get the block, and add a reference to it in the page vector and the * textBlocks vector */ textBlock = new StringBuffer(pageHTML.substring(low,i)); page.add(textBlock); textBlocks.add(textBlock); return i; }//getTextBlock /** * This method returns the index one past the end of the tag * in the HTML source that starts at index low, and adds the * tag therein to the page vector. * Since this method already has to parse a given tag, we'll * also let it construct the tag object, as well as the internal * attribute objects for that tag. Doing so makes the method a little * messier, but saves quite a bit of time for long pages. * * @param pageHTML a String containing the source HTML * @param low an int, the starting index for our search * @param an int, the index 1 past the tag's end */ private int getTag(String pageHTML, int low) { int size = pageHTML.length(); int high = pageHTML.indexOf(">",low); if ( high == -1 ) return pageHTML.length(); // shouldn't happen, 'cept for crappy HTML /* get the index of the next space, the end of the tag's name */ int spaceIndex = pageHTML.indexOf(" ",low); if ( spaceIndex > high || spaceIndex == -1 ) spaceIndex = high; /* if there's a newline before the space, stop the name at the newline */ int newlineIndex = pageHTML.indexOf("\n",low); if ( spaceIndex > newlineIndex && newlineIndex != -1) spaceIndex = newlineIndex; Vector tag = new Vector(); String tagName = pageHTML.substring(low+1,spaceIndex).toLowerCase(); tag.add(tagName); addTagToHash(tag); /* ignore stuff between */ if ( tagName.startsWith("script") ) { }//if /* ignore HTML comments */ if (tagName.startsWith("!--")){ Vector v = getCommentText(pageHTML, low); tag.add(new StringBuffer((String) v.lastElement())); /* in case we want to modify the comment text */ page.add(tag); return ((Integer) v.firstElement()).intValue(); }//if /* handle weird DTD tags, like !DOCTYPE, etc */ if ( tagName.startsWith("!") ) { int endOfTag = pageHTML.indexOf(">",spaceIndex+1); StringBuffer textBuff = new StringBuffer(pageHTML.substring(spaceIndex+1,endOfTag)); textBuff.insert(0," "); tag.add(textBuff); page.add(tag); return endOfTag+1; }//if int cur = low; int eqSign = 0; while ( (eqSign = pageHTML.indexOf("=",cur)) < high && eqSign != -1 ){ Vector attribute = new Vector(); /* allow for values starting immediately after a newline */ int prevSpace = pageHTML.lastIndexOf(" ",eqSign); int prevNewline = pageHTML.lastIndexOf("\n",eqSign); if ( prevSpace < prevNewline ) prevSpace = prevNewline; attribute.add((pageHTML.substring(prevSpace + 1,eqSign)).toLowerCase()); /* figure out what denotes the end of this attribute */ char quote = pageHTML.charAt(eqSign+1); if ( quote != '\'' && quote != '\"' ) quote = ' '; /* find the end of the quote */ int endQuote = pageHTML.indexOf(quote,eqSign+2); int newline = pageHTML.indexOf("\n",eqSign+2); /* */ if ( quote == ' ' && endQuote > newline && newline != -1 ) { endQuote = newline; }//if /* we're switching links to use target='_top', so if we find an "a" tag with * a target attribute, skip the attribute and continue */ if ( ((String) attribute.firstElement()).equals("target") && ((String) tag.firstElement()).equals("a") ) { high = pageHTML.indexOf(">", endQuote+1); cur = endQuote + 1; continue; }//if /* last attribute is unquoted, so no more looping */ if ( quote == ' ' && endQuote > high ) { String attributeValue = pageHTML.substring(eqSign+1, high); /* let's think about supporting the BASE tag */ if ( tagName.equals("base") && ((String) attribute.firstElement()).equals("href") ) { int lastSlash = attributeValue.lastIndexOf("/"); if ( lastSlash > 7 ) docRoot = attributeValue.substring(0,lastSlash+1); else docRoot = attributeValue.toString(); }//if attribute.add(new StringBuffer("\"" + attributeValue + "\"")); addAttributeToHash(attribute); tag.add(attribute); break; }//if /* fail gracefully when there's a weird exception here with the string indices going wacko */ try{ if ( quote == ' ' ) attribute.add(new StringBuffer("\"" + pageHTML.substring(eqSign+1,endQuote) + "\"")); else attribute.add(new StringBuffer(pageHTML.substring(eqSign+1,endQuote+1))); }catch(StringIndexOutOfBoundsException sioobe) { System.out.println("HTMLPage: getTag: Caught StringIndexOutOfBoundsException. Finishing up."); break; }//catch StringBuffer attributeValue = (StringBuffer) attribute.lastElement(); /* let's think about supporting the BASE tag */ if ( tagName.equals("base") && ((String) attribute.firstElement()).equals("href") ) { int lastSlash = attributeValue.lastIndexOf("/"); if ( lastSlash > 7 ) docRoot = attributeValue.substring(0,lastSlash+1); else docRoot = attributeValue.toString(); }//if /* hash the attribute */ addAttributeToHash(attribute); /* add the attribute to the current tag */ tag.add(attribute); /* update the loop indices */ high = pageHTML.indexOf(">",endQuote+1); cur = endQuote+1; }//while /* add the target attribute to "a" tags */ if ( ((String) tag.firstElement()).equals("a") ) { Vector targetAttribute = new Vector(); targetAttribute.add("target"); targetAttribute.add(new StringBuffer("'_top'")); tag.add(targetAttribute); addAttributeToHash(targetAttribute); }//if page.add(tag); //System.out.println("tag = '"+tag+"'"); return ++high; }//getTag /** * Put the given attribute object in the attributes hash, keyed with * its name. If there's already a value for that key, add the attribute * to the vector already hashed. Otherwise, make a new vector and add * the attribute there. * * @param attribute an HTMLAttribute to be added to the page's hash */ private void addAttributeToHash(Vector attribute) { //System.out.println("Adding attribute to hash..."); /* get the vector of attributes hashed with the same name */ //Vector atrsVec = (Vector) attributes.get(attribute.getName()); Vector atrsVec = (Vector) attributes.get((String)attribute.firstElement()); /* if none has been hashed yet, make a new vector and hash it */ if ( atrsVec == null ) { atrsVec = new Vector(); attributes.put((String)attribute.firstElement(),atrsVec); }//if atrsVec.add(attribute); // append this attribute to the vector }//addAttributeToHash /** * Put the given tag object in the tag hash, keyed with * its name. If there's already a value for that key, add the tag * to the vector already hashed. Otherwise, make a new vector and add * the tag there. * * @param tag an HTMLTag to be added to the page's hash */ private void addTagToHash(Vector tag) { //System.out.println("Adding tag to hash..."); /* get the vector of tag hashed with the same name */ //Vector tagsVec = (Vector) tags.get(tag.getName()); Vector tagsVec = (Vector) tags.get((String)tag.firstElement()); /* if none has been hashed yet, make a new vector and hash it */ if ( tagsVec == null ) { tagsVec = new Vector(); //tags.put(tag.getName(),tagsVec); tags.put((String)tag.firstElement(),tagsVec); }//if tagsVec.add(tag); // append this tag to the vector }//addTagToHash /** * Return the index after the comment, and the comment text as a String, * in the form of a Vector (multiple return values would be nice, but we're * not using Scheme). In this context, we consider the comment text to be * everything between the first space in the tag and the tag-close delimiter. * * @param pageHTML a String with the HTML text we're parsing * @param low the index of the start of this comment tag * @return a vector containing the index after this tag and the text of the comment */ private Vector getCommentText(String pageHTML, int low) { //System.out.println("Getting comment text..."); int end = pageHTML.indexOf("--",low+5); // find the ending "--" delimiter //int end = pageHTML.indexOf(">",low); // find the ending "--" delimiter if ( end == -1 ) end = pageHTML.length(); end = pageHTML.indexOf(">",end); // the end of the comment is the next ">" Vector output = new Vector(); output.add(new Integer(++end)); // one past the end of the tag output.add(pageHTML.substring(low + 5,end - 1)); //the comment text doesn't include the ">" return output; }//getTagName /** * Returns a vector of references to the tags with the given name string. * * @param tagName the String name of the tags we want access to * @return a vector of references to all tags with name = tagName */ public Vector getTagsByName(String tagName){ return (Vector) tags.get(tagName); }//getTagsByName /** * Returns a vector of references to the attributes with the given name string. * * @param tagName the String name of the tags we want access to * @return a vector of references to all tags with name = tagName */ public Vector getAttributesByName(String attributeName){ return (Vector) attributes.get(attributeName); }//getTagsByName /** * Returns a Vector containing references to all plain text StringBuffers within * this page, allowing direct modification of the page's text. * * @return a Vector containing references to all the text blocks within this page */ public Vector getTextBlocks(){ return textBlocks; }//getTextBlocks /** * Replace any text matching the given regular expression with the given replacement. * New and improved version, now with Enumerations (my fondness for While loops * notwithstanding)!!! * * @param pattern a String containing a regex * @param replacement a String containing replacement text */ public void replaceText(String pattern,String replacement) { /* replace plain text */ System.out.println("Replacing plain text."); for ( Enumeration e = textBlocks.elements() ; e.hasMoreElements() ; ) { StringBuffer text = (StringBuffer) e.nextElement(); text.replace(0,text.length(),(text.toString()).replaceAll(pattern,replacement)); }//while /* replace alt text */ System.out.println("Replacing alt text."); Vector altAttributes = (Vector) attributes.get("alt"); // get the alt attributes if ( altAttributes == null ) return; /* loop over alt attributes, replacing text in their value stringbuffers */ for ( Enumeration f = altAttributes.elements() ; f.hasMoreElements() ; ) { Vector attribute = (Vector) f.nextElement(); System.out.println("attribute = '"+attribute+"'"); StringBuffer altText = (StringBuffer) attribute.lastElement(); altText.replace(0,altText.length(),(altText.toString()).replaceAll(pattern,replacement)); }//while }//replaceText /** * Insert the given text at the top of this page, immediately after the <'body'> tag. * * @param textToInsert a String containing text to insert into this page */ public void insertAtTop(String textToInsert){ topExtra.insertElementAt(new StringBuffer(textToInsert),0); }//insertAtTop /** * Insert the given tag at the top of this page, immediately after the <'body'> tag. * * @param tagToInsert a Vector containing an html tag to insert into this page */ public void insertAtTop(Vector tagToInsert){ topExtra.insertElementAt(tagToInsert,0); }//insertAtTop /** * Insert the given text at the bottom of this page, immediately before the <'/body'> tag. * * @param textToInsert a String containing text to be added to this page */ public void insertAtBottom(String textToInsert){ bottomExtra.insertElementAt(new StringBuffer(textToInsert),0); }//insertAtBottom /** * Insert the given HTML tag at the bottom of this page, immediately before the <'/body'> tag. * * @param tagToInsert a Vector representing an HTML tag to be added to this page */ public void insertAtBottom(Vector tagToInsert){ bottomExtra.insertElementAt(tagToInsert,0); }//insertAtBottom /** * Insert the given HTML tag in the header, immediately after the <'head'> tag. * * @param tagToInsert an HTMLTag to be added to this page */ public void insertInHead(Vector tagToInsert){ headExtra.insertElementAt(tagToInsert,0); }//insertInHead(HTMLTag tagToInsert) /** * Convert any relative URLs in this HTMLPage to their absolute equivalents, and * then prepend the virtual URL for gathering a virtual URL to all URLs. This method * essentially makes one pass over all src, action, and href attributes in the attributes * hash table, inserting whatever text necessary. * * @param request the HTTPRequest corresponding to this page * @param response the HTTPResponse corresponding to this page * @param prefix a string containing the text to insert before the absolute URL * @param banner a boolean indicating whether a flag should be inserted to signal that the banner should be inserted */ public void makeURLsAbsolute(HTTPRequest request, HTTPResponse response) { // /* if we get a response with a null entityText field, we have problems... */ // System.out.println("Making urls absolute"); // if (response.entityText == null) return; // //// if (request.uriString.equalsIgnoreCase("http://webraveler2.banner.bannerhtml")) return; //// //// /* figure out the document root and server root */ //// int thirdSlash = request.uriString.indexOf("/",7); //// //// if ( thirdSlash == -1 ) { //// request.uriString += "/"; //// thirdSlash = request.uriString.length()-1; //// }//if //// //// /* the prefix for URLs that start with "/" */ //// serverRoot = request.uriString.substring(0,thirdSlash); //// //// /* Fix for uris with /~name */ //// if (request.uriString.length() > thirdSlash+1 && request.uriString.charAt(thirdSlash+1) == '~') { //// int fourthSlash = request.uriString.indexOf('/',thirdSlash+1); //// if ( fourthSlash == -1 ) { //// request.uriString += "/"; //// fourthSlash = request.uriString.length()-1; //// } //if //// serverRoot = request.uriString.substring(0,fourthSlash); // }//if // // /* it's nice to know where the last slash in the URL is, so that // * we know the base for URLs that don't start with "/" */ // int lastSlash = request.uriString.lastIndexOf("/"); // if ( lastSlash == 6 ) { // request.uriString += "/"; // lastSlash = request.uriString.length(); // }//if // // /* this string is the prefix for URLs that don't start with "/" or "http://" */ // docRoot = request.uriString.substring(0,lastSlash+1); // // /* make URLs absolute for the following attribute names */ // handleAttribute("href"); // // handleAttribute("src"); // // /* this should be updated to somehow check for method="POST" within the current tag, // * and not add a vurl to the corresponding action attribute */ // handleAttribute("action"); // // //response.entityText = toString(); // // //response.entityBytes = response.entityText.getBytes(); // //response.localHeaders.put("content-length",""+response.entityBytes.length); // }//makeURLsAbsolute /** * Return the text blocks following 'p' tags, as a vector. * * @return a Vector containing all the paragraphs in the html document */ public Vector getParagraphs(){ Vector paragraphs = new Vector(); int count = 0; int size = page.size(); while ( count < size ) { Vector tag; try { tag = (Vector) page.elementAt(count++); /* is this a "
" tag? */ if ( ((String)tag.firstElement()).equalsIgnoreCase("p") ) { /* YES. get the next block of text */ int bound = 15; /* max number of page elements to look at before we give up */ StringBuffer textBlock = new StringBuffer(); while ( count < size && bound-- > 0) { try { Vector tagTwo = (Vector) page.elementAt(count++); if ( ((String)tagTwo.firstElement()).equalsIgnoreCase("/p") ){ break; //quit at the explicit end of the paragraph }//if continue;//go to the next iteration if it's a different tag } catch (ClassCastException cce) { count--; //wasn't a vector, so reset the counter }//catch try { textBlock.append(((StringBuffer) page.elementAt(count++)).toString()); }catch (ClassCastException cce) { } }//while paragraphs.add(textBlock.toString()); }//if /* NO. go to next tag. */ } catch ( ClassCastException cce) {} }//while return paragraphs; }//getParagraphs /** * Changes the action attribute of the form tag to an absolute URL. * If the method is POST, the virtual url prefix is added, otherwise * the url remains unmodified so that the banner frame will be added * when it is requested * * @param textToInsert a string containing the virtual url text to insert */ public void handleForms(String textToInsert) { /* handle forms */ Object item; StringBuffer itemStr; Vector tag; boolean post = false; int count = 0; int size = page.size(); while ( count < size ) { item = page.elementAt(count++); try { itemStr = (StringBuffer) item; }catch (ClassCastException cce) { /* must be a vector... */ tag = (Vector) item; /* is this a "