rendered paste bodyimport BeautifulSoup
import twisted.web.client
import twisted.internet
import urllib2
class unjobs:
baseurl = "http://unjobs.org/duty_stations"
duty_stations = []
positions = []
station_pages = []
def __init__(self):
self.fetchStations()
self.fetchPages()
def fetchStations(self):
front = BeautifulSoup.BeautifulSoup(urllib2.urlopen(self.baseurl).read())
mytable = front.html.body.table.contents[3].contents[3].contents[5].table.tr.findAll('a')
for i in mytable:
station_name = i.renderContents()
station_url = i.attrs[1][1]
self.duty_stations.append({'url': station_url,'name': station_name})
def appendPage(self, content):
self.station_pages.append(content)
def failurePage(self, content):
twisted.internet.reactor.stop()
def fetchPages(self):
deferred_coll = []
for i in self.duty_stations:
temp = twisted.web.client.getPage(i['url'].encode('utf-8'), timeout = 30)
deferred_coll.append(temp)
for i in deferred_coll:
i.addCallback(self.appendPage)
i.addErrback(self.failurePage)
twisted.internet.reactor.run()
twisted.internet.reactor.stop()
def printStations(self):
print self.duty_stations
print len(self.station_pages)
def jobs_run():
x = unjobs()
x.printStations()
jobs_run()