Miscellany

import urllib2, os, threading, time, sysURL = "http://img.4chan.org/b/imgboard.html" #"http://img.4chan.org/b/res/53264202.html"imgdir = "_4chan"I = 0Y = 0T = 0recurse = Falsedef log(msg):    print "[%s]: %s" % (time.strftime("%I:%M"),msg)if "-r" in sys.argv or "-recurse" in sys.argv or "-watch" in sys.argv:    log( "recursing enabled" )    recurse = Trueelse:    log( "recursing disabled" )def threadurl(thread):    nurl = "http://img.4chan.org/b/res/%s" % thread    #log( "threadurl: %s" % nurl )    return nurlclass FetchThread(threading.Thread):    def __init__(self, topic):        self.topic = topic        threading.Thread.__init__(self)    def run(self):        global I        log( "fetching topic %s" % self.topic )        s=urllib2.urlopen(self.topic);html=s.read();s.close()        g = html.split("File :<a href=\"")                for split in g:            sp = split.split("\"")[0]            if sp[0:4] == "http":                I += 1                bname = os.path.basename(sp)                localfile = os.path.join(imgdir,bname)                if not os.path.exists(localfile):                    log( "fetching %s to %s (%i of %i)" % (bname,localfile,I,len(g)) )                    try:                        st = urllib2.urlopen(sp);f = open(localfile,'wb')                        f.write(st.read());f.close();st.close()                    except Exception, e:                        log( "exception in FetchThread: %s (url: %s)" % (e,sp) )                    class MainGET(threading.Thread):    def run(self):        global T        log( "got main page" )        FetchThread(URL).start()        try:            if recurse: s=urllib2.urlopen(URL);html=s.read();s.close()        except Exception, e:            log("MainGET: exception: %s (url: %s)" % (e,url))            return                if recurse:            g = html.split("<a href=\"res/")            for split in g:                sp = split.split("#")[0]                if sp[len(sp)-4:len(sp)] == "html":                    T += 1                    #log( "MainGET: res ends with html, getting topic. (%i)" % T)                    FetchThread(threadurl(sp)).start()                #log("end MainGET")                    class MainWatcher(threading.Thread):    def run(self):        global Y        while True:            Y += 1            MainGET().start()            log( "updating main page: %i" % Y )            time.sleep(30)def main():    MainWatcher().start()if __name__ == '__main__': main()