import os,reimport urllib,sqlite3,unicodedataimport sys,shutil#regex expressions for parsecomicdate_re=re.compile('\(([0-9]{4})\)')title_re=re.compile('\(\w*?\)|\([0-9]{4}\)')clean_re=re.compile('[^\w ]')#regex_expressions for sqlite functionsFS_re=re.compile('(\)|\])(\(|\[)')fixnum_re=re.compile('(\d+)( |\()?([Oo][Ff]|\/) ? \d+')#########################################parsecomic class########################################class parsecomic: def __init__(self,fname, Title_Split=False, FS=' '): #set name, extension #fname: full file name unomodified, name=name - extension, ext = extension d={'rar':'cbr','zip':'cbz','gz':'cbt','tar':'cbt','tbz':'cbt','bz2':'cbt'} self.fname=fname self.name='.'.join(fname.split('.')[:-1]) self.ext=fname.split('.')[-1].lower() if d.has_key(self.ext): self.ext=d[self.ext] #set varibales #working: whats remaining in the filename (we hack away at this as we find matches) #FS:field seprator (default = space) #title,volume,number,credits, date all self explanatory. self.working=self.name self.FS=FS self.TS=Title_Split self.lowest_match=len(self.fname) self.title=None self.volume=None self.number=None self.credits=None self.date=None self.annual=False self.dbmatch=False #call the parseall function (this is the function that ties everything togather) self.parseall() def popregex(self,regex, ALL=None): """Process the regex expression, popping the matche defualt process last""" #checks the regex expression against working, if it finds it #it returns the match and deletes that field from self.working #note: the match is formated correctly, the field is just everything #between the FS #pass true to all, to have it get all matching fields to the regex expression #this is usefull if you have multiple [credits] [credits] field and want #to make a list of them #IMPORTANT: the function returns group 1, so make sure to put the data u want #in () in the regex expression matchlist=[] match_re=re.compile(regex) for field in reversed(self.split_working): match=match_re.search(field) if match: matchlist.append(match.group(1)) if self.working.find(field) and self.working.find(field)<self.lowest_match: self.lowest_match=self.working.find(field) self.split_working.remove(field) self.working=self.FS.join(self.split_working) if not ALL: break if not matchlist: return else: if not ALL: return matchlist[0] return list(set(matchlist)) def parseall(self): """returns components of the comic filename""" #inital= things to be done before we begin processing other things #essentially this fixes weird stuff "fixed"...etc.. self.initial() #parse each field self.split_working=self.working.split(self.FS) self.parse_date() self.parse_credits() self.parse_number() self.parse_volume() self.parse_title() self.check_db() self.final() def initial(self): """Modifications before parsing data""" #insert FS between paren if their stuck togather: )( => ) ( self.working=FS_re.sub('\g<1>'+self.FS+'\g<2>', self.working) #convert #n of n', n/n, n (of n) => n self.working=fixnum_re.sub('\g<1>',self.working) #remove bad words self.remove_field_containing('fixed') self.remove_field_containing('TPB') self.remove_field_containing('UNKNOWNVAL',1) if self.remove_field_containing('annual'): self.annual=True def final(self): """Modifications after everything is done""" if self.annual:self.title+=' - Annual' def remove_field_containing(self, s, matchcase=0): """Remove any field containing s""" found=False split_working=self.working.split(self.FS) for field in split_working[:]: if (s in field) or (not matchcase and s.lower() in field.lower()): split_working.remove(field) self.working=self.FS.join(split_working) found=True return found def parse_date(self): """Get the year""" #match any 4digit number in () or []. we make an assumption that this is the date. if not self.date: self.date=self.popregex("^[\(\[]([0-9]+?)[\)\]]$") #match dd-dd-dd{2,4} in () [] assuming thats thats the date if not self.date: self.date=self.popregex("^[\(\[](\d{2}-\d{2}-\d{2,4})[\)\]]$") def parse_credits(self): """Get the credits""" #grab anything thats between () or [] #run self.date() first to avoid having date saved as credits if not self.credits: self.credits=self.popregex("[\[|\(](.+?)[\]|\)]",1) def parse_number(self): """Get the Number""" #find any number starting with # if not self.number: self.number=self.popregex("^#([0-9]+)$") #find any number, period if not self.number: self.number=self.popregex("^([0-9]+)$") if self.number: self.number=self.number.zfill(3) def parse_volume(self): """Get the Volume""" #find any number that starts with v or V if not self.volume: self.volume=self.popregex("^[vV]([1-9]+)$") def parse_title(self): """Get the title""" #join the remaining fields as the title if not self.title: self.title=self.working[:self.lowest_match].strip().rstrip('- ') #if Title split, then split the title using ' - ' if self.title and self.TS: self.title=self.title[:self.lowest_match].split(' - ')[0].strip() def check_db(self): """Checks db for right info""" con=getcon() results=None if self.title: results=search_comic(self.title,con) if results: self.dbmatch=True self.title=results[0][0] bad=[': ','/'] for i in bad: self.title=self.title.replace(i,' - ') if not self.date and self.volume: self.date=results[int(self.volume)-1][1] if not self.volume and self.date: self.volume=[i[0] for i in enumerate(results) if self.date>=i[1][1]] if self.volume: self.volume=self.volume[-1]+1 def fprint(self): """Print the date in a nice format to stdout""" print self.fname print "-"*len(self.fname) print "title : %s\nvolume : %s\nnumber : %s\nyear : %s\ncredit : %s\next : %s" % (self.title, self.volume, self.number, self.date, self.credits, self.ext) print def getdata(self): """return a tuple of the data""" return self.title, self.volume, self.number, self.date, self.credits, self.ext def getstring(self): """Return a string of the new filename""" data=list(self.getdata()) if data[4]:data[4]=')('.join(data[4]) data=[field or "UNKNOWNVAL" for field in data] return '%s v%s %s (%s) (%s).%s' % tuple(data)#########################################SQL functions########################################def downloadlist(): print "downloading data" url=urllib.urlopen('http://comicbookdb.com/browse.php?search=Title&letter=all') data=url.read() url.close() find_list=re.findall('<a href=\"title.php\?ID=[0-9]+\" >(.+?)<',data,re.MULTILINE) find_list=[fix_ref(title) for title in find_list] print "done!" return find_listdef getcon(): f=None if not os.path.isfile('.comic.db'): con=sqlite3.connect('.comic.db') con.execute("create table titles_table(title text,date integer(4),keywords text)") con.execute("create index keywords_date_idx on titles_table(keywords,date)") make_db_list(downloadlist(), con) print "creating db" con.commit();con.close() print "done" con=sqlite3.connect('.comic.db') con.text_factory=str return condef fix_ref(s): """Converts & to & and such""" d={'&':'&'} for key in d.keys(): if key in s: s=s.replace(key,d[key]) s=re.sub('\&[A-Za-z0-9]+?;','',s) # convert to ascii s=convert_to_ascii(s) return sdef convert_to_ascii(s): d={'\xbc':'1/4', '\xbd':'1/2', '\xbe':'3/4'} for i in d.keys(): if i in s: s=s.replace(i,d[i]) return unicodedata.normalize('NFKD',unicode(s,'latin')).encode('ascii','ignore')def make_db_list(comic_list,con): """writes title, date, keywords to sql db""" dblist=[] for title in comic_list: date=int(date_re.search(title).group(1)) title=date_re.sub('',title)# title=title_re.sub('',title) title=title.strip() keywords=getkeywords(title) con.execute("insert into titles_table values(?,?,?)",(title,date,keywords))def getkeywords(s): """Gets the keywords out of a string""" #initiate string s=s.lower() s=s.center(len(s)+2) #bad will be delted, space will be turned into ' ' bad=[' the ',' and ',' comics '] space=['/','_'] for i in space: if i in s: s=s.replace(i,' ') for i in bad: if i in s: s=s.replace(i,' ') s=clean_re.sub('',s) t=' '.join(list(set([v for v in s.split() if len(v)>2 or v.isdigit()]))) if not t: t=' '.join(list(set([v for v in s.split()]))) s=t return sdef search_comic(query,con): """Search sql_db for comic""" keywords_list=[getkeywords(query)] keywords_list+=[getkeywords(q) for q in query.split(' - ') if ' - ' in query] if not keywords_list:return [] for keywords in keywords_list: found=con.execute("select title, date from titles_table where keywords=? order by date",(keywords,)).fetchall() if found: return found#########################################User Interface Functions########################################def get_movelist(directory): """Gets the move list [[old,new],[old,new]] from directory""" movelist=[] filelist=os.listdir(directory) if sys.argv[0] in filelist: filelist.remove(sys.argv[0]) for entry in filelist: e=parsecomic(entry) movelist.append([entry,e.getstring(),e.dbmatch]) movelist.sort(key=lambda x:x[1]) return movelistdef get_movelist_file(f): """Gets the movelist [[old,new],[old,new]] from file""" movelist=[] lines=[] for entry in f: if entry=='\n':continue entry=entry.split(':',1) lines.append(int(entry[0].strip())) entry=entry[1] if '===>' in entry: movelist.append([x.strip() for x in entry.split('===>')]+[True]) if '???>' in entry: movelist.append([x.strip() for x in entry.split('???>')]+[False]) return set(lines),movelistdef movefiles(movelist): """move the files renaming them""" directory=getdir() for old, new,dbmatch in movelist: old='%s/%s'%(directory,old) new='%s/%s'%(directory,new) shutil.move(old,new) sys.exit()def menu(movelist): """Asks the user what he wants to do""" pad=80 reply='' for old,new,dbmatch in movelist: if dbmatch: print '%s ===> %s' %(old.ljust(pad),new) else: print '%s ???> %s' %(old.ljust(pad),new) print '\nKEY:\n===> Found in DB\n???> Not found in DB' while reply.lower() not in ['r','e','q']: reply=raw_input("[r]ename, [e]dit, [q]uit:").lower().strip() if reply=='q':sys.exit() elif reply=='e': return run_edit(movelist) elif reply=='r': movefiles(movelist)def run_edit(movelist): """runs vim letting the user edit the data""" original=movelist[:] pad=80 while True: f=open('.movelist','w') for i,(old,new,dbmatch) in enumerate(movelist): if dbmatch: f.write(str(i).ljust(5)+': '+old.ljust(pad)+'===> '+new+'\n\n') else: f.write(str(i).ljust(5)+': '+old.ljust(pad)+'???> '+new+'\n\n') f.close() os.system('vim .movelist') f=open('.movelist') lines,new_movelist=get_movelist_file(f) f.close() os.unlink('.movelist') bad=set(range(len(movelist)))-lines if movelist==new_movelist:break if bad: bad=sorted(list(bad))[::-1] for i in bad: del movelist[i] del original[i] for i,d in enumerate(new_movelist): if not movelist[i][1]==d[1]: new_movelist[i][2]=False if not movelist[i][0]==d[0]: e=parsecomic(d[0]) new_movelist[i][1]=e.getstring() new_movelist[i][2]=e.dbmatch movelist=new_movelist movelist.sort(key=lambda x:x[1]) for i,d in enumerate(movelist): original[i][1]=d[1] original[i][2]=d[2] return originaldef getdir(): """Returns the working directory name""" if not len(sys.argv)==2: print 'Usage %s [dirname]' % os.path.basename(sys.argv[0]) sys.exit() directory=sys.argv[1] if not os.path.isdir(directory): print "'%s' is not a directory" %(directory) sys.exit() return directory#########################################User Interface Program########################################movelist=get_movelist(getdir())while True: movelist=menu(movelist)