Stuff

#!/usr/bin/python

import urllib, urllib2, re
from urlparse import urlparse
from sgmllib import SGMLParser

class FetchLinks(SGMLParser):
	'''Use this + SGMLParser to fetch links from website'''
	
	def reset(self):
		SGMLParser.reset(self)
		self.urls = []

	def start_a(self, attrs):
		href = [v for k, v in attrs if k=='href']
		if href:
			self.urls.extend(href)


def linkCheck(link):
	'''Process links to add host or strip javascript out'''

	if link[0] == 'j':
		p = re.compile(".*?javascript:.*?\'(http:.*?)\'.*", re.IGNORECASE)
		URL = p.sub(r'\1', link)
		if URL.find('http') == -1:
			URL = ''	
		else:
			return URL

	
	if link[0] == '/':
		URL = rootURL + link
		
		return URL
	else:
		URL = link
		return URL
		



rootURL = 'http://guide.real.com'

openURL = urllib.urlopen(rootURL)
fetchURLs = FetchLinks()
fetchURLs.feed(openURL.read())
openURL.close()
fetchURLs.close()



for URL in fetchURLs.urls:
	try:
		print URL
		print linkCheck(URL)
		urllib2.urlopen(linkCheck(URL))
	except urllib2.HTTPError, e:
		if e.code == 401:
			print 'Not Authorized'
		if e.code == 404:
			print 'Not Found'
		if e.code == 503:
			print 'Service Unavailable'
	else:
		print 'Succeeded'