Mine

#!/usr/bin/env pythonimport sysfrom BeautifulSoup import BeautifulSoupfrom subprocess import Popen, PIPEimport multiprocessingimport Queuen_documents = 0all_docs = []def get_document(addr):    global n_documents    global all_docs    fname = "d%03d.html" % n_documents    p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)    out, err = p.communicate()    if p.wait():        print '--- out ---'        print out        print '--- err ---'        print err        raise Exception("Error calling wget")    n_documents += 1    print fname, addr    all_docs.append(fname)    return fnamedef get_picture(fname, addr):    p = Popen(['wget', '-O', fname, addr], stdout=PIPE, stderr=PIPE)    out, err = p.communicate()    if p.wait():        print '--- out ---'        print out        print '--- err ---'        print err        raise Exception("Error calling wget")def main():    addr  = 'http://guesshermuff2.blogspot.com/'        # get all html    while True:        fname = get_document(addr)        soup = BeautifulSoup(open(fname).read())        optag = None        for a in soup.findAll('a'):            if a.text == 'Older Posts':                optag = a                break        if not optag:            break        addr = optag['href']        work_queue = multiprocessing.Queue()    jobs = 0    for fname in all_docs:        work_queue.put(fname)        jobs += 1        class Worker(multiprocessing.Process):        '''        Woker to find all .jpg hrefs in a document        '''        def __init__(self, work_queue, result_queue):            multiprocessing.Process.__init__(self)            self.work_queue   = work_queue            self.result_queue = result_queue            self.kill_received = False        def run(self):            while not self.kill_received:                # get a task:                try:                    job = self.work_queue.get_nowait()                except Queue.Empty:                    break                # the actual processing                print job                soup = BeautifulSoup(open(job).read())                pic_links = soup.findAll('a', attrs={'href': lambda x: x and x.lower().endswith('.jpg')})                pic_links = [pl['href'] for pl in pic_links]                self.result_queue.put((job, pic_links))        class PictureWorker(multiprocessing.Process):        def __init__(self, work_queue, result_queue):            multiprocessing.Process.__init__(self)            self.work_queue   = work_queue            self.result_queue = result_queue            self.kill_received = False        def run(self):            while not self.kill_received:                # get a task:                try:                    fname, link = self.work_queue.get_nowait()                except Queue.Empty:                    break                # the actual processing                print fname, link                get_picture(fname, link)                self.result_queue.put(fname)        result_queue = multiprocessing.Queue()    for i in range(multiprocessing.cpu_count()):        worker = Worker(work_queue, result_queue)        worker.start()        # collect results    results = []    while len(results) < jobs:        results.append(result_queue.get())        n_pics = 0    work_queue = multiprocessing.Queue()    for fname, links in sorted(results):        print '===== %s =====' % fname        for link in links:            print '   ', link            work_queue.put(("p%08d.jpg" % n_pics, link))            n_pics += 1        result_queue = multiprocessing.Queue()    for i in range(multiprocessing.cpu_count()):        worker = PictureWorker(work_queue, result_queue)        worker.start()        results = []    while len(results) < n_pics:        results.append(result_queue.get())        print "done?"if __name__ == '__main__':    sys.exit(main())