# minimalist Web-Spider with threadsimport sets, urllib, re, urlparse, thread# Used for threadingdef process_queue(id): global queue, visited while True: while not queue: pass url = queue.pop(0) print 'Thread %d Doing %s ' % (id, url) try: sock = urllib.urlopen(url) page = sock.read() sock.close() except IOError: continue for match in re.findall(r'href=\"(.+?)\"', page): candidate = urlparse.urljoin(url, match) if candidate in visited: continue queue.append(candidate) visited.add(candidate)queue = ['http://www.oregonstate.edu']visited = sets.Set()for i in range(20): thread.start_new_thread(process_queue, (i,))while True: pass