#!/usr/bin/env python# dupecheck v0.3""" A quick dupe-checker. Will first compare file-sizes and if some match, will then check their md5 sums. Won't compute md5sums until they're needed."""import md5import stringimport osimport timesizes = {}class FileRef: """ Lazilly evaluate md5 sum of file """ def __init__(self, fname, md5 = None): self.fname = fname self.md5 = md5 def md5val(self): if self.md5 == None: self.md5 = md5file(open(self.fname, "rb")) return self.md5def md5file(f): """Returns an md5sum hex string of a file.""" m = md5.new() while 1: bytes = f.read(8192) if not bytes: break m.update(bytes) return m.hexdigest()class DupeChecker: def __init__(self, actions, verbose=True): self.actions = actions def execute(self): checked=0 for f in open('/dev/stdin', 'r'): f = string.rstrip(f) # chomp fstats = os.stat(f) checked = checked + 1 if not (checked / 250): print ".", # If already know of a file with the same size if sizes.has_key(str(fstats.st_size)): oldfs = sizes[str(fstats.st_size)] newmdsum = md5file(open(f, "rb")) found = False for realf in oldfs: if realf.md5val() == newmdsum: found = True if self.actions and len(self.actions) > 0: for a in self.actions: if a == 'Print': print "Dupe! %d\t%s (first was: %s)" % (fstats.st_size, f, realf.fname) elif a == 'Delete': os.remove(f) else: print '!', # If didn't remove it, must be a new file with existing size if not found: #os.path.exists(f): #print "%s New hash for an old filesize (%d)" % (newmdsum, fstats.st_size) oldfs.append( FileRef(f, newmdsum) ) else: #print "%d New filesize!" % fstats.st_size sizes[str(fstats.st_size)] = [] sizes[str(fstats.st_size)].append(FileRef(f, None))def main(): from optparse import OptionParser optparser = OptionParser() optparser.add_option("-d", "--delete", dest="actions", action="append_const", const="Delete", help="delete all non-unique files encountered (caution!)") (options, args) = optparser.parse_args() if not options.actions: options.actions = [] options.actions.append('Print') d = DupeChecker(options.actions) start = time.time() d.execute() stop = time.time() difftime = stop - start print "\nDone! %d seconds elapsed." % (difftime)if __name__ == "__main__": main()