All pastes #2060284 Raw Edit

Mine

public text v1 · immutable
#2060284 ·published 2011-05-15 13:44 UTC
rendered paste body
#!/usr/bin/env python

import sys
from BeautifulSoup import BeautifulSoup
from subprocess import Popen, PIPE
import multiprocessing
import Queue

n_documents = 0
all_docs = []

def get_document(addr):
    global n_documents
    global all_docs
    fname = "d%03d.html" % n_documents
    p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)
    out, err = p.communicate()
    if p.wait():
        print '--- out ---'
        print out
        print '--- err ---'
        print err
        raise Exception("Error calling wget")
    n_documents += 1
    print fname, addr
    all_docs.append(fname)
    return fname

def get_picture(fname, addr):
    p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)
    out, err = p.communicate()
    if p.wait():
        print '--- out ---'
        print out
        print '--- err ---'
        print err
        raise Exception("Error calling wget")

def main():
    addr  = 'http://guesshermuff2.blogspot.com/'
    
    # get all html
    while True:
        fname = get_document(addr)
        soup = BeautifulSoup(open(fname).read())
        optag = None
        for a in soup.findAll('a'):
            if a.text == 'Older Posts':
                optag = a
                break
        if not optag:
            break
        addr = optag['href']
    
    work_queue = multiprocessing.Queue()
    jobs = 0
    for fname in all_docs:
        work_queue.put(fname)
        jobs += 1
    
    class Worker(multiprocessing.Process):
        '''
        Woker to find all .jpg hrefs in a document
        '''
        def __init__(self, work_queue, result_queue):
            multiprocessing.Process.__init__(self)
            self.work_queue   = work_queue
            self.result_queue = result_queue
            self.kill_received = False
        def run(self):
            while not self.kill_received:
                # get a task:
                try:
                    job = self.work_queue.get_nowait()
                except Queue.Empty:
                    break
                # the actual processing
                print job
                soup = BeautifulSoup(open(job).read())
                pic_links = soup.findAll('a', attrs={'href': lambda x: x and x.lower().endswith('.jpg')})
                pic_links = [pl['href'] for pl in pic_links]
                self.result_queue.put((job, pic_links))
    
    class PictureWorker(multiprocessing.Process):
        def __init__(self, work_queue, result_queue):
            multiprocessing.Process.__init__(self)
            self.work_queue   = work_queue
            self.result_queue = result_queue
            self.kill_received = False
        def run(self):
            while not self.kill_received:
                # get a task:
                try:
                    fname, link = self.work_queue.get_nowait()
                except Queue.Empty:
                    break
                # the actual processing
                print fname, link
                get_picture(fname, link)
                self.result_queue.put(fname)
    
    result_queue = multiprocessing.Queue()
    for i in range(multiprocessing.cpu_count()):
        worker = Worker(work_queue, result_queue)
        worker.start()
    
    # collect results
    results = []
    while len(results) < jobs:
        results.append(result_queue.get())
    
    n_pics = 0
    work_queue = multiprocessing.Queue()
    for fname, links in sorted(results):
        print '===== %s =====' % fname
        for link in links:
            print '   ', link
            work_queue.put(("p%08d.jpg" % n_pics, link))
            n_pics += 1
    
    result_queue = multiprocessing.Queue()
    for i in range(multiprocessing.cpu_count()):
        worker = PictureWorker(work_queue, result_queue)
        worker.start()
    
    results = []
    while len(results) < n_pics:
        results.append(result_queue.get())
    
    print "done?"

if __name__ == '__main__':
    sys.exit(main())