rendered paste body#!/usr/bin/env python
import sys
from BeautifulSoup import BeautifulSoup
from subprocess import Popen, PIPE
import multiprocessing
import Queue
n_documents = 0
all_docs = []
def get_document(addr):
global n_documents
global all_docs
fname = "d%03d.html" % n_documents
p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.wait():
print '--- out ---'
print out
print '--- err ---'
print err
raise Exception("Error calling wget")
n_documents += 1
print fname, addr
all_docs.append(fname)
return fname
def get_picture(fname, addr):
p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.wait():
print '--- out ---'
print out
print '--- err ---'
print err
raise Exception("Error calling wget")
def main():
addr = 'http://guesshermuff2.blogspot.com/'
# get all html
while True:
fname = get_document(addr)
soup = BeautifulSoup(open(fname).read())
optag = None
for a in soup.findAll('a'):
if a.text == 'Older Posts':
optag = a
break
if not optag:
break
addr = optag['href']
work_queue = multiprocessing.Queue()
jobs = 0
for fname in all_docs:
work_queue.put(fname)
jobs += 1
class Worker(multiprocessing.Process):
'''
Woker to find all .jpg hrefs in a document
'''
def __init__(self, work_queue, result_queue):
multiprocessing.Process.__init__(self)
self.work_queue = work_queue
self.result_queue = result_queue
self.kill_received = False
def run(self):
while not self.kill_received:
# get a task:
try:
job = self.work_queue.get_nowait()
except Queue.Empty:
break
# the actual processing
print job
soup = BeautifulSoup(open(job).read())
pic_links = soup.findAll('a', attrs={'href': lambda x: x and x.lower().endswith('.jpg')})
pic_links = [pl['href'] for pl in pic_links]
self.result_queue.put((job, pic_links))
class PictureWorker(multiprocessing.Process):
def __init__(self, work_queue, result_queue):
multiprocessing.Process.__init__(self)
self.work_queue = work_queue
self.result_queue = result_queue
self.kill_received = False
def run(self):
while not self.kill_received:
# get a task:
try:
fname, link = self.work_queue.get_nowait()
except Queue.Empty:
break
# the actual processing
print fname, link
get_picture(fname, link)
self.result_queue.put(fname)
result_queue = multiprocessing.Queue()
for i in range(multiprocessing.cpu_count()):
worker = Worker(work_queue, result_queue)
worker.start()
# collect results
results = []
while len(results) < jobs:
results.append(result_queue.get())
n_pics = 0
work_queue = multiprocessing.Queue()
for fname, links in sorted(results):
print '===== %s =====' % fname
for link in links:
print ' ', link
work_queue.put(("p%08d.jpg" % n_pics, link))
n_pics += 1
result_queue = multiprocessing.Queue()
for i in range(multiprocessing.cpu_count()):
worker = PictureWorker(work_queue, result_queue)
worker.start()
results = []
while len(results) < n_pics:
results.append(result_queue.get())
print "done?"
if __name__ == '__main__':
sys.exit(main())