All pastes #956044 Raw Edit

dup-files

public text v1 · immutable
#956044 ·published 2008-03-25 00:17 UTC
rendered paste body
#!/usr/bin/perl
use strict;
use warnings;
use File::Find;
use Digest::MD5;

###############################################################
#find_dups(@dir_list) -- Return an arrey containing a list
#       of duplicate files.
###############################################################
sub find_dups(@)
{
        # The list of directories to search
        my @dir_list = @_;

        # If nothing there, return nothing
        if ($#dir_list < 0) {
            return (undef);
        }

        my %files;  # Files indexed by size

        # Go through the file tree and find all
        # files with a similar size
        find( sub {
                -f &&
                push @{$files{(stat(_))[7]}}, $File::Find::name
            }, @dir_list
        );

        my @result = ();   # The resulting list

        # Now loop through the list of files by size and see
        # if the md5 is the same for any of them
        foreach my $size (keys %files) {
             if ($#{$files{size}} < 1) {
                next;
             }
             my %md5;       # MD5 -> file name array hash

             # Loop through each file of this size and
             # compute the MD5 sum
             foreach my $cur_file (@{$files{$size}}) {
                # Open the file. Skip the files we can't open
                open(FILE, $cur_file) or next;
                binmode(FILE);
                push @{$md5{
                    Digest::MD5->new->addfile(*FILE)->hexdigest}
                }, $cur_file;
                close (FILE);
             }
             # Now check for any duplicates in the MD5 has
             foreach my $hash (keys %md5) {
                if ($#{$md5{$hash}} >= 1) {
                   push(@result, [@{$md5{$hash}}]);
                }
           }
        }
        return @result
}

my @dups = find_dups(@ARGV);

foreach my $cur_dup (@dups) {
    print "Duplicates\n";
    foreach my $cur_file (@$cur_dup) {
        print "\t$cur_file\n";
    }
}