rendered paste bodyimport BeautifulSoupimport twisted.web.clientimport twisted.internetimport urllib2class unjobs: baseurl = "stations" duty_stations = [] positions = [] station_pages = [] def __init__(self): self.fetchStations() self.fetchPages() def fetchStations(self): front = BeautifulSoup.BeautifulSoup(urllib2.urlopen(self.baseurl).read()) mytable = front.html.body.table.contents[3].contents[3].contents[5].table.tr.findAll('a') for i in mytable: station_name = i.renderContents() station_url = i.attrs[1][1] self.duty_stations.append({'url': station_url,'name': station_name}) def appendPage(self, content): self.station_pages.append(content) def failurePage(self, content): twisted.internet.reactor.stop() def fetchPages(self): deferred_coll = [] for i in self.stations: temp = twisted.web.client.getPage(i['url'].encode('utf-8'), timeout = 30) deferred_coll.append(temp) for i in deferred_coll: i.addCallback(self.appendPage) i.addErrback(self.failurePage) twisted.internet.reactor.run() twisted.internet.reactor.stop() def printStations(self): print self.stations print len(self.station_pages)def jobs_run(): x = () x.printStations()jobs_run()