All pastes #399849 Raw Edit

African swallow velo

public text v1 · immutable
·published 2007-03-18 09:35 UTC
rendered paste body
import BeautifulSoup
import twisted.web.client
import twisted.internet
import urllib2

class unjobs:

	baseurl		= "http://unjobs.org/duty_stations"
	duty_stations	= []
	positions	= []
	station_pages	= []

	def __init__(self):
		self.fetchStations()
		self.fetchPages()

	def fetchStations(self):
		front = BeautifulSoup.BeautifulSoup(urllib2.urlopen(self.baseurl).read())
		mytable = front.html.body.table.contents[3].contents[3].contents[5].table.tr.findAll('a')
		for i in mytable:
			station_name	= i.renderContents()
			station_url	= i.attrs[1][1]
			self.duty_stations.append({'url': station_url,'name': station_name})

	def appendPage(self, content):
		self.station_pages.append(content)

	def failurePage(self, content):
		twisted.internet.reactor.stop()

	def fetchPages(self):
		deferred_coll = []
		for i in self.duty_stations:
			temp = twisted.web.client.getPage(i['url'].encode('utf-8'), timeout = 30)
			deferred_coll.append(temp)
		for i in deferred_coll:
			i.addCallback(self.appendPage)
			i.addErrback(self.failurePage)
		twisted.internet.reactor.run()
		twisted.internet.reactor.stop()


	def printStations(self):
		print self.duty_stations
		print len(self.station_pages)




def jobs_run():
	x = unjobs()
	x.printStations()

jobs_run()