dupecheck.py

#!/usr/bin/env pythonimport osfrom os.path import join, getsize, getmtimeimport sysfrom hashlib import sha1def size_duplicates (paths):	sizes = {}	for path in paths:		size = getsize (path)		if size in sizes:			yield path, sizes[size]		else:			sizes[size] = path			def hash_duplicates (duplicates):	hashes = {}	for path, original_path in duplicates:		original_hash = sha1 (file (original_path).read ()).hexdigest ()		hashes.setdefault (original_hash, original_path)				hash = sha1 (file (path).read ()).hexdigest ()		if hash in hashes:			yield path, hashes[hash]		else:			hashes[hash] = path			def scan (path):	for root, dirs, files in os.walk (path):		paths = (join (root, filename) for filename in files)		size_dups = size_duplicates (paths)		hash_dups = hash_duplicates (size_dups)		for dup in hash_dups:			yield dup			def main (argv):	if len (argv) < 2:		print 'Usage: %s <path>' % argv[0]		return 1	if '--delete' in argv:		for dup_path, original_path in scan (argv[1]):			print "Unlinking %s" % dup_path			os.unlink (dup_path)	else:		for dup_path, original_path in scan (argv[1]):			print "DUPLICATE FILE"			print "   %s" % dup_path			print "of %s" % original_path			print ""			if __name__ == '__main__':	sys.exit (main (sys.argv))