/** * Class WebCrawler takes a URL * and goes through it grabbing * other web addresses and emails * * @author Zach B. * @version 21 April 2008 */import java.net.URL;import java.net.MalformedURLException;import java.net.URLConnection;import java.io.BufferedReader;import java.io.BufferedInputStream;import java.io.InputStreamReader;import java.io.IOException;import java.util.Set;import java.util.HashSet;import java.util.Iterator;import java.util.regex.Pattern;import java.util.regex.Matcher;public class WebCrawler{ private URL url; private Set<String> crawledAddr = new HashSet<String>(), waitingAddr = new HashSet<String>(), emails = new HashSet<String>(), badurls = new HashSet<String>(); private Iterator<String> waitingIter = this.waitingAddr.iterator(); private StringBuffer buffer = new StringBuffer(""); private Window gui; public WebCrawler() { } public WebCrawler(String urlString, Window gui) { // May need to add if (this.isValidURL(urlString)) if always getting exceptions // new URL will throw a MalformedURLException which can be caught this.gui = gui; try { this.url = new URL(urlString); } catch (MalformedURLException mue) { this.gui.getInfoLabel().setText("Invalid URL"); } }//#################################################################### ACCESSOR/MUTATOR METHODS ###################################################// /** * Returns a Set which contains all the emails * keyed together by the address they were found at. */ public Set getEmails() { return this.emails; } /** * Returns a Set which contains the addresses * that you wanted. */ public Set getCrawledAddresses() { return this.crawledAddr; } public Set getWaitingAddresses() { return this.waitingAddr; } public Iterator getWaitingIterator() { return this.waitingIter; } public Set getBadURLS() { return this.badurls; } /** * Returns the source of the current URL (only works after a call to readData) */ public StringBuffer getBuffer() { return this.buffer; } /** * Returns the current URL */ public URL getURL() { return this.url; } public void setURL(String urlString) { try { this.url = new URL(urlString); } catch(MalformedURLException mue) { this.gui.getInfoLabel().setText("Invalid URL"); } }//#################################################################### REAL METHODS ###################################################// /** * Checks to see if the given String is a valid URL. */ public boolean isValidURL(String urlString) { URL test;// javax.swing.JLabel label = this.gui.getInfoLabel(); javax.swing.JTextPane label = this.gui.getStatusPane(); try { test = new URL(urlString); URLConnection connection; BufferedReader data; try { // Opens the connection connection = test.openConnection(); connection.connect(); // Allows the data to be read data = new BufferedReader(new InputStreamReader(new BufferedInputStream(connection.getInputStream()))); } catch (IOException ioe) { label.setText("Invalid URL(doesn't go anywhere): " + urlString); if (!this.badurls.contains(urlString)) this.badurls.add(urlString); return false; } } catch (MalformedURLException mue) { label.setText("Malformed URL: " + urlString); this.badurls.add(urlString); return false; } return true; } /** * readData allows the program to access the data * in the file. It uses regular expressions to find emails * and web addresses and stores them in the appropriate Sets. */ public void readData() {// javax.swing.JLabel label = this.gui.getInfoLabel(); javax.swing.JTextPane label = this.gui.getStatusPane(); URLConnection connection; BufferedReader data; String line; try { // Opens the connection connection = this.url.openConnection(); connection.connect(); System.out.println("Connection opened..."); // Allows the data to be read data = new BufferedReader(new InputStreamReader(new BufferedInputStream(connection.getInputStream()))); System.out.println("Reading data...(May take a while)"); // Reads the file line by line while ((line = data.readLine()) != null) { // Finds emails this.findEmails(line); // Finds addresses this.findSites(line); // Updates this.buffer with the current source of this.url this.buffer.append(line + "\n"); } this.crawledAddr.add(this.url.toString()); this.waitingAddr.remove(this.url.toString()); // And update the iterator so we can get the next URL to go to this.waitingIter = this.waitingAddr.iterator(); // And take the next URL off of waitingAddr (using the waitingIter) this.advanceURL(); label.setText("Reading done."); System.out.println("Reading done."); data.close(); } catch (IOException ioe) { label.setText("IO Error: " + ioe.getMessage()); } } private void findEmails(String line) { String group; Pattern emailPattern = Pattern.compile("[A-Za-z0-9\\.] + (\\[at\\] | \\s\\[at\\]\\s | \\s\\@\\s | \\@) + [A-Za-z0-9\\.] + (\\[dot\\] | \\s\\[dot\\]\\s | \\s\\.\\s | \\.) + [A-Za-z0-9\\.]+", Pattern.COMMENTS); Matcher m = emailPattern.matcher(line); if (m.find()) { // Grabs the email group = m.group(); // Formats it into name@provider.domain group = Pattern.compile("\\[at\\] | \\s\\[at\\]\\s | \\s\\@\\s", Pattern.COMMENTS).matcher(group).replaceAll("@"); group = Pattern.compile("\\[dot\\] | \\s\\[dot\\]\\s | \\s.\\s", Pattern.COMMENTS).matcher(group).replaceAll("."); // Adds email to email set if (!this.emails.contains(group)) this.emails.add(group); } } private void findSites(String line) { String group; Pattern addrPattern = Pattern.compile("( (http://www + \\.) | (http://) | (www + \\.) | ( \\s ) | ( \" | = ) ) + [A-Za-z0-9\\.] + \\. + [A-Za-z0-9\\.]+", Pattern.COMMENTS); Matcher m = addrPattern.matcher(line); if (m.find()) { // Grabs the address (and trims whitespace) group = m.group().trim(); group = Pattern.compile(" (=) | (\") | (=\") ", Pattern.COMMENTS).matcher(group).replaceAll(""); group = group.trim(); // if group doesn't have any beginning http or www, consider it a link to another page of the same site. if (!group.contains("www.") && !group.contains("http://")) { group = this.url.getHost() + "/" + group; } // If the url doesn't have the needed protocol, add it if (!group.contains("http://")) { group = "http://" + group; } group = group.toLowerCase(); if (!this.crawledAddr.contains(group) && !this.waitingAddr.contains(group) && !this.badurls.contains(group) && this.isValidURL(group)) this.waitingAddr.add(group); } } private void advanceURL() { if (this.waitingIter.hasNext()) { String urlS = this.waitingIter.next(); if (!this.badurls.contains(urlS) && this.isValidURL(urlS)) { try { this.url = new URL(urlS); } catch (MalformedURLException mue){} } else { this.waitingAddr.remove(urlS); this.waitingIter = this.waitingAddr.iterator(); } } }}