Untitled

/** This class will download the data from the website.
 * 
 */
package wj;
import org.htmlparser.filters.*;
import org.htmlparser.*;
import org.htmlparser.util.*;
import java.io.*;
import java.net.*;

/**
 * @author chotchki
 * 
 */
public class DataScraper {
	private DateString generator;

	private Config config;

	private Storage storage;

	/**
	 * 
	 * @param currentDate
	 *            Date that you wished scraped
	 */
	public DataScraper(Config config) {
		this.config = config;
		this.generator = new DateString(this.config);
		this.storage = new Storage(this.config);
	}

	public Storage getData() {
		while (!this.generator.isToday()) {
			CustomCalendar day = this.generator.getDate();
			//System.out.println(day.getDateString());
			if(this.storage.checkDate(day)){
				this.generator.nextDay();
				continue;
			}
			double djiClose = 0;
			double nyseAdvancers = 0; 
			double nyseDecliners = 0;
			boolean empty = false;
			String pageText = "";
			try {
				URL connection = new URL(this.config.URL
						+ this.generator.getDate().getDateString());
				BufferedReader in = new BufferedReader(
						new InputStreamReader(
								connection.openStream()));
				String inputLine;
				while ((inputLine = in.readLine()) != null){
					pageText += inputLine;
				}
				in.close();
				//Parse out all the values from the page if any values fail
				//Then make sure the day is set as empty			        
				Parser parser = new Parser(pageText);
				djiClose = this.parseDOW(parser);
				Parser parser2 = new Parser(pageText);
				nyseAdvancers = this.parseNYSEA(parser2);
				Parser parser3 = new Parser(pageText);
				nyseDecliners = this.parseNYSED(parser3);
			}catch(Exception e){
				empty = true;
			}
			if(empty == true){
				this.storage.addEmptyDate(day);
			}
			else{
				this.storage.addDate(day, djiClose, nyseAdvancers, nyseDecliners);
			}
			this.storage.save();
			this.generator.nextDay();
		}
		return this.storage;
	}

	public double parseDOW(Parser parser) throws NumberFormatException {
		String closeDirty = "";
		try {
			// This grabs the closing value for the DOW Jones
			NodeList list = new NodeList();
			NodeFilter filter = new AndFilter(
					new TagNameFilter("TD"),
					new HasSiblingFilter(
							new HasChildFilter(
									new StringFilter(
											"Dow Jones"
									)
							)
					)
			);
			for (NodeIterator e = parser.elements(); e.hasMoreNodes();) {
				e.nextNode().collectInto(list, filter);
			}
			closeDirty = list.elementAt(7).toPlainTextString();

		} catch (Exception e) {
		}

		double closeClean = Double.parseDouble(closeDirty);
		return closeClean;
	}
	public double parseNYSEA(Parser parser) throws NumberFormatException {
		String closeDirty = "";
		try {
			// This grabs the closing value for the DOW Jones
			NodeList list = new NodeList();
			NodeFilter filter = 
				new AndFilter(
						new TagNameFilter("TD"),
						new HasSiblingFilter(
								new HasChildFilter(
										new StringFilter(
												"Advancers"
										)
								)
						)
				);
			for (NodeIterator e = parser.elements(); e.hasMoreNodes();) {
				e.nextNode().collectInto(list, filter);
			}
			closeDirty = list.elementAt(7).toPlainTextString();

		} catch (Exception e) {

		}

		double closeClean = Double.parseDouble(closeDirty);
		return closeClean;
	}
	public double parseNYSED(Parser parser) throws NumberFormatException {
		String closeDirty = "";
		try {
			NodeList list = new NodeList();
			NodeFilter filter = 
				new AndFilter(
						new TagNameFilter("TD"),
						new HasSiblingFilter(
								new HasChildFilter(
										new StringFilter(
												"Decliners"
										)
								)
						)
				);
			for (NodeIterator e = parser.elements(); e.hasMoreNodes();) {
				e.nextNode().collectInto(list, filter);
			}
			closeDirty = list.elementAt(6).toPlainTextString();

		} catch (Exception e) {
			e.printStackTrace();
		}

		double closeClean = Double.parseDouble(closeDirty);
		return closeClean;
	}
}