RUBY SPIDER :D

require 'open-uri'
require 'net/http'
require 'string-edit'
require 'Crawler'
require 'uri'
#===============================================================================
# & WebSpider
#----------------------------------------------------------------------------------------------
# Download Files from sites, for quick access, threaded. Automatically
#===============================================================================
class WebSpider

  #------------------------------------------------------------------
  # * Initialize web spider
  #------------------------------------------------------------------
  def initialize(site)
    @links = {}
    @images = []
    @site = site
    time1 = Time.now
    explore(@site)
    while running?
      begin
      links = @links
      links.each_key {|link|
        unless @links[link]
          explore(link)
        end
      }
      rescue RuntimeError
        sleep 0.0001
      end
    end
    time2 = Time.now
    done = time2 - time1
    File.open('Details.txt','a') do |file|
      file.write "#{@site} finished in #{done} seconds.. \n"
    end
  end
  
  def running?
    @links.each_value do |value|
      return true if value == false
    end
    return false
  end
  
  def explore(addr)
    begin
      puts "[!] Exploring #{addr}"
      links = WebCrawler.get_hrefs(addr)
      links.each do |link|
        next unless WebCrawler.same_site(link,@site)
        @links[link] = false unless @links.has_key?(link)
      end
      images = WebCrawler.get_imgs(addr)
      images.each do |image|
        @images << image unless @images.include?(image)
      end
      @links[addr] = true
      @images.each do |image|
        Thread.new {
          WebCrawler.download(image)
        }
      end
      @images.clear
    rescue Errno::EINVAL, Errno::ENOENT, SocketError, URI::InvalidURIError, Errno::EBADF, OpenURI::HTTPError, OpenSSL::SSL::SSLError, RuntimeError
      return
    end
  end

end