rendered paste body#!/usr/bin/env rubyrequire 'rubygems'require 'db'require 'mechanize'categories_id = ARGV.join(',')categories = Category.find(:all, :conditions => "id IN (#{categories_id})")categories.each do |c| puts "Parsing category #{c.name}\n" search = c.name.downcase.gsub('_', '+').gsub(' ', '+') feed_links = [] first_search = 'http://search.msn.com/feeds/results.aspx?first=1&q=' + search puts "Parsing #{first_search} for count pages..." agent = WWW::Mechanize.new page = agent.get(first_search) for link in page.links #feed_links << link.attributes['href'] unless link.attributes['gping'].nil? end results = (page/:h5)[0].all_text.scan(/Page\ 1\ of\ (.*)\ results/)[0][0].gsub(',','').to_i puts "There are #{results.to_s} pages" results.times do |i| next if i < 10 next if i < 200 if i.modulo(10) == 1 url = 'http://search.msn.com/feeds/results.aspx?first=' + i.to_s + '&q=' + search begin puts "Parsing #{url}..." sleep(30) page = agent.get('http://search.msn.com/feeds/results.aspx?first=' + i.to_s + '&q=' + search) rescue url = 'http://search.msn.com/feeds/results.aspx?first=' + i.to_s + '&q=' + search puts "There was a problem parsing #{url} retrying..." retry end for link in page.links feed_links << link.attributes['href'] unless link.attributes['gping'].nil? end end end feed_links.each { |f| puts " Adding feed #{f} to the db...\n" feed = Feed.new feed.category_id = c.id feed.url = f feed.save }end