#!/usr/bin/perl
#-------------------------
# Title: find_duplicate_files.pl
# Purpose: Find Duplcate files
# How: search a directory recursively for files
# compare md5 of each file
# output files with same md5 in orgnaized fassion.
#
# Uses a hash of arrays
#
# Example of how data will be stored:
# %unique = (
# 9a4688f7fc2dceaa20d97fe14c606619 => [ "file1", "file2" ],
# e768cef480108cae41e6b76ba2efa4a1 => [ "file3", "file4", "file5" ],
# 8f6ca31f6295dd897c6df305f5df9203 => [ "file6", "file7", "file8" ],
# );
#
#-------------------------
#
# load the modules, Load The Modules, LOAD THE MODULES
#
use File::Find; # used to traverse directory
use Digest::MD5; # use to find md5 of each file
#
# 1=print extra output, 0=more silent
#
my $debug=0;
#
# hash containing all data, md5 is the key, value array of file names.
#
my %unique=();
my $ref_unique=\%unique;
#
# If no arguments are given, use current dir
#
@ARGV = (".") unless @ARGV;
#
# Function calcuates md5 on files
# and populates the %unique hash using $ref_unique reference
#
sub md5sum {
#
# Unless a file, exit function
#
return unless -f ;
#
# Assign file name to variable
#
my $file="$File::Find::name";
#
# Open file
#
open(FILE, qq{$_} ) or die "Can't open $file: $!";
binmode(FILE);
#
# Calculate md5sum
#
my $number=Digest::MD5->new->addfile(*FILE)->hexdigest ;
#
# Print values if debug is on;
#
if ( $debug == 1 ){ print "$number:$file\n"; }
#
# Now Add this info to the hash %unique
#
if ( exists $$ref_unique{$number} ){
#
# If the md5 has been seen before, append to hash elements array
#
push @{ $$ref_unique{"$number"} }, "$file" ;
} else {
#
# Otherwise Start a new hash element
# And add the file name to the array
#
my @array = ( $file );
$$ref_unique{$number} = [ ( $file ) ];
}
}
#
# Finally call find
#
find(\&md5sum, @ARGV);
#
# Print out the results
#
foreach my $x ( keys %unique ) {
my $y = $#{$unique{$x}} + 1;
print "$x $y times\n";
for my $i ( @{ $unique{$x} } ){
print "\t$i\n";
}
}