Anonymous

/*
 * Crawler.java
 *
 * Created on 22. November 2006, 19:45
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package javaapplication5;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;

/**
 *
 * @author kj09
 */
public class Crawler extends Thread { //Holt sich den Quelltext, gibt ihn an den Op und extrahiert die Links.
    /** Creates a new instance of Crawler */
    public Crawler(final Workpool pool, final Storage stor) throws MalformedURLException, IOException {
            
       while(pool.isEmpty() == false || pool.crawlers.activeCount() != 0){
           yield();
        URLentry urls = pool.get();
        URL url = new URL(urls.getadress());
        final String source = getCode(url);
         System.out.println("Processing..."+url.toString());
        
        Operator Op = new Operator(source, pool, stor, urls.getID());
                   
        
        //Extraction der Links:
        //String[] Links = extractLinks(source);
        System.out.println("Processed "+url.toString()+"...");
       }
       System.out.println("Crawler shutdown! Running crawlers: "+pool.crawlers.activeCount()+" List empty? : "+pool.isEmpty());
       yield();
     
    }
    
    private String[] extractLinks(String source){ //Extrahiert die Links aus dem Dokument
     Collection output = new ArrayList();
        String pattern =  "(?:[hH][rR][eE][fF]\\s*=)(?:[\\s\"\"']*)(?!#|[Mm]ailto|[lL]ocation.|[jJ]avascript|.*css|.*this\\.)(.*?)(?:[\\s>\"\"'])"; //Der Regex zum Extrahieren der Links.
        String Results[] = source.split(pattern);
     
        return(Results);
   }
    
   private String readout(InputStream is) throws IOException { //Liest einen Stream in einen Array.
        StringBuffer sb = new StringBuffer();
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        String line;
        while ((line = br.readLine()) != null)
            sb.append(line);
        return sb.toString();
    }
    
    private String getCode(URL url){ //Holt den Quelltext ab
        InputStream contstream = null;
        String output = null;
        try {
            contstream = (InputStream) url.openStream();
        } catch (IOException ex) {
            System.out.println("Error while downloading "+url+".");
            ex.printStackTrace();
        }
        try {
              
            output = readout(contstream);
        } catch (IOException ex) {
            System.out.println("Error while downloading "+url+".");
            ex.printStackTrace();
        }
        
        return(output);
    }
}