sPyder

import sysimport reimport urllib,urllib2import stringimport datetimeanchor_match = re.compile(r'.*?<a.*?href="(.*?)".*?>(.*?)</a>.*?')#ommited hrefs (apache)ommit = ['?C=N;O=D','?C=S;O=A','?C=D;O=A','?C=M;O=A','/']buff = ''dump_files = Truedef usage():    print """USAGE: sPyder url_to_spider target_file [-f]\n\turl_to_spider\tURI you want to spider\t\t\tNOTE: if URI contains spaces you will have to put the URI string into ''\t\t\tor replace the spaces with %20 \ttarget_file\tfile where dir structure will be stored in\n\tOPTIONAL parameters:\t-f\t\tno files written in dump\n\tEXAMPLES:\t\tsPyder http://someserver.com/directory c:/httpdump -f\t\tsPyder 'http://someserver.com/directory with space' c:/myshit/httpdump\t\tsPyder http://someserver.com/directory%20with%20space c:/httpdump -f -s"""def readIndexFile(uri = '',level = 0):    global buff,ommit,anchor_match    request = urllib2.Request(uri)      try:        response = urllib2.urlopen(request)      except urllib2.HTTPError, e:        print 'ERR: ('+str(e.code)+') Error occured. Current URI:'+uri        buff += '\n|-'+'--'*level+uri[uri[:-1].rfind('/')+1:]+' ERR: ('+str(e.code)+')'    except urllib2.URLError, e:        print 'ERR: Failed to reach the URI ('+str(e.reason[0])+':'+e.reason[1]+')'        buff += '\n|-'+'--'*level+uri[uri[:-1].rfind('/')+1:]+' ERR: ('+str(e.reason[0])+':'+e.reason[1]+')'    else:        for match in re.finditer(anchor_match,response.read()):            if match.group(1) not in ommit and match.group(2) != 'Parent Directory':                decoded_uri = urllib.unquote(match.group(1))                if match.group(1)[-1] == '/':                    if level > 0: buff += '\n| '+'  '*level+'|-'+decoded_uri                    else: buff += '\n|-'+'--'*level+decoded_uri                    readIndexFile(uri+match.group(1),level+1)                else:                    if(dump_files):                        if level > 0: buff += '\n| '+'  '*level+'| '+decoded_uri                        else: buff += '\n| '+'|--'*level+decoded_uri#script startif(len(sys.argv) <3):    print 'ERR: Lack of parameters'    usage()else:    try:        f = open(sys.argv[2], 'w')    except:        print 'Directory does not exist or file not writable: '+sys.argv[2]    else:        if len(sys.argv) == 4 and sys.argv[3] == '-f': dump_files = False        print 'Spidering URI.....'        if sys.argv[1][-1] != '/': sys.argv[1] = sys.argv[1]+'/'        readIndexFile(sys.argv[1])        print 'Writing to file...'        f.write('sPyder 1.1 by starenka <oggova@starenka.net>')        f.write('\n@'+datetime.datetime.now().strftime("%A %d.%m.%Y %H:%M:%S")+'\n\n'+sys.argv[1])           f.write(buff)        print buff+'\n'        print 'Done ;)'