Stuff

/*
 * Crawler.java
 *
 * Created on 22. November 2006, 19:45
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package javaapplication5;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;

/**
 *
 * @author kj09
 */
public class Crawler extends Thread { //Holt sich den Quelltext, gibt ihn an den Op und extrahiert die Links.
    
    /** Creates a new instance of Crawler */
    public Crawler(String urls, final Workpool pool, final Storage stor, final CrawlerController crawlercontroller) throws MalformedURLException, IOException {
        crawlercontroller.checkin();
        if(urls == ""){  } else {
        
        System.out.println("Processing...");
        URL url = new URL(urls);
        final String source = getCode(url);
        
               
        Operator Op = new Operator(source, pool, stor, crawlercontroller);
                   
        
        //Extraction der Links:
        String[] Links = extractLinks(source);
        
        System.out.println("Processed "+url+"...");
              
        }
    }
    
    private String[] extractLinks(String source){ //Extrahiert die Links aus dem Dokument
     Collection output = new ArrayList();
        String pattern =  "(?:[hH][rR][eE][fF]\\s*=)(?:[\\s\"\"']*)(?!#|[Mm]ailto|[lL]ocation.|[jJ]avascript|.*css|.*this\\.)(.*?)(?:[\\s>\"\"'])"; //Der Regex zum Extrahieren der Links.
        String Results[] = source.split(pattern);
     
        return(Results);
   }
    
   private String readout(InputStream is) throws IOException { //Liest einen Stream in einen Array.
        StringBuffer sb = new StringBuffer();
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        String line;
        while ((line = br.readLine()) != null)
            sb.append(line);
        return sb.toString();
    }
    
    private String getCode(URL url){ //Holt den Quelltext ab
        InputStream contstream = null;
        String output = null;
        try {
            contstream = (InputStream) url.openStream();
        } catch (IOException ex) {
            System.out.println("Error while downloading "+url+".");
            ex.printStackTrace();
        }
        try {
              
            output = readout(contstream);
        } catch (IOException ex) {
            System.out.println("Error while downloading "+url+".");
            ex.printStackTrace();
        }
        
        return(output);
    }
}