/*
* Main.java
*
* Created on 22. November 2006, 15:04
*
* To change this template, choose Tools | Template Manager
* and open the template in the editor.
*/
package javaapplication5;
import java.applet.Applet;
import java.awt.Color;
import java.awt.Frame;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.regex.Matcher;
import javax.swing.JApplet;
import javax.swing.JTextArea;
import jxl.Workbook;
import jxl.write.Label;
import jxl.write.WritableSheet;
import jxl.write.WritableWorkbook;
import jxl.write.WriteException;
/**
*
* @author kj09
*/
public class Main extends Applet {
public long crawlers=0;
public long crawlersOP=0;
private WritableWorkbook workbook;
/** Creates a new instance of Main */
public void init(){
System.out.println("Welcome.");
System.out.println("Starting up...");
//0. Interface erzeugen:
System.out.println("..Building User Interface..");
JTextArea text = new JTextArea();
text.setEditable(false);
text.setBackground(new Color(255, 255, 255));
text.setForeground(new Color(0, 0,0 ));
this.add(text);
this.validate();
System.out.println("..Creating Storage Device..");
//1. Erstellen eines Docs zum Speichern der anfallenden Ergebnisse:
try {
final Storage stor = new Storage("C:/Serienbrief_Adressen.xls", "data");
System.out.println("Done");
//2. Initialisieren des Pools:
Collection initvals = new ArrayList();
System.out.println("..Initializing URL-Pool..");
final Workpool p = new Workpool();
System.out.println("Done");
System.out.println("..Generating inital-URLs..");
int i;
for( i=116233; i<116246; i++){
p.add("http://www.bildungsportal.nrw.de/BP/_suche1/SchuleSuchen/schule_info.php?SNR="+i);
}
System.out.println("Done");
//OK, nun kann die Arbeit beginnen.
//Main-Schleife initialisieren:
final int crawlerlimit = 10;
System.out.println("*Crawlers allowed: "+crawlerlimit+"*");
System.out.println("..Starting main-process..");
int y = 0;
final boolean isRunning = true;
final Main crawlercontrol = this;
while(p.isEmpty()==false && isRunning == true){//Wärend unser Pool also noch Daten zu liefern hat...
System.out.println("Crawlers running: "+getWorkingCrawlers());
if(getWorkingCrawlers()<crawlerlimit){ //Wenn die Crawleranzahl das Limit nicht übersteigt...
System.out.println("OK, add a new Crawler. Running actually:"+getWorkingCrawlers());
//Füttern wir regelmäßig unsere Crawler.
//Starten eines neuen Threads:
new Thread(){
public void run(){
try {
Crawler crawl = new Crawler(
p.get() ,
stor,
crawlercontrol
);
} catch (MalformedURLException ex) {
ex.printStackTrace();
}
}
}.start();
}
}
System.out.println("Done.");
System.out.println("...Waiting for the crawlers to be finished...");
new Thread(){ public void run(){
long ival = 0;
long eval = 0;
while(getWorkingCrawlers() > 0){
yield();
ival = getWorkingCrawlers();
if(ival < eval){ System.out.println("Crawlers left: "+getWorkingCrawlers()); }
eval = getWorkingCrawlers();
}
System.out.println("Done.");
try {
stor.close();
} catch (IOException ex) {
ex.printStackTrace();
}
}}.start();
} catch (IOException ex) {
System.out.println("Failed while creating Storage-Device.");
ex.printStackTrace();
}
}
public synchronized void checkin(){
System.out.println("Crawler is checking in!");
crawlers++;
}
public synchronized void checkout(){
System.out.println("Crawler is checking out!");
crawlers--;
}
public synchronized long getWorkingCrawlers(){
return(crawlers);
}
}