Untitled

#!/usr/bin/ruby


require 'net/http'
require 'uri'

def fetch(urls)

  threads = []

  for url in urls
    threads << Thread.new(url) { |url|
      uri = URI.parse(url)
      path = uri.path + "?" + uri.query
      puts "Fetching: #{url} - #{uri.host} - #{path}"
      h = Net::HTTP.new(uri.host, 80)
      resp, data = h.get(path, nil )
      puts "Got #{url}:  #{resp.message} #{data.length}"
      idx = 0
      idx = url.rindex('/') if ! url.rindex('/').nil?
      file = url[idx + 1, url.length - 1].gsub(/[\?\=\&]/, "_")
      page = File.new(file + ".html", "w")
      page << data if !data.nil?
      page.close
    }
  end
  threads.each { |aThread|  aThread.join }
end


then I create an array with something like

http://mysite/page?1
http://mysite/page?2
http://mysite/page?3

and fetch the pages. In my logs I see

Fetching: http://mysite/page?id=1 - www.civiweb.com - /page?id=1
Fetching: http://mysite/page?id=2 - www.civiweb.com - /page?id=2
Fetching: http://mysite/page?id=3 - www.civiweb.com - /page?id=3
Got http://mysite/page?id=3:  OK 37392
Got http://mysite/page?id=3:  OK 34008
Got http://mysite/page?id=3:  OK 34403


notice that the data.length is correct but the url isn't. Because of this all pages are saved under the same file.