require 'page'require 'page_queue'require 'repository'require 'set'# (extend to use drb to provide arbitrary function calls # to the daemon?)module SoMark class Crawler attr_reader :queue, :repository def initialize @threads = Set.new @debug = true end def debug(msg) puts msg if @debug or Thread.current[:debug] end def start(options) @queue = options[:queue] if options[:queue] @repository = options[:repository] if options[:repository] @freq = options[:freq] if options[:freq] @freq ||= 0.1 @max_queue = options[:max_queue] if options[:max_queue] @max_queue ||= 20000 @max_thread = options[:max_thread] if options[:max_thread] @max_thread ||= 500 while true if @threads.size < @max_thread shift_page end sleep @freq end @repository.log "end of queue", "error" end def shift_page @threads << Thread.new do begin page = @queue.shift return if not page if @repository.saved? page if @queue.size == 0 @queue.push(*page.links.collect {|l| Page.new(l)}) end else if @repository.saving page download(page) end if @queue.size > @max_queue @queue.clean end ensure @threads.delete Thread.current end end end def download(page) if page.read @repository.save page pages = (page.links.collect { |url| linked_page = Page.new(url) }) pages = @repository.unsaved pages @queue.push(*pages) else # requeue failure @queue << page end end endend