WTC Scrape

#!/usr/bin/perl

use strict;
use CGI::Carp qw /fatalsToBrowser/;
use LWP::UserAgent;

my $url         = "http://graphics8.nytimes.com/packages/html/nyregion/20050812_WTC_GRAPHIC/met_WTC_histories_full_01.html";
my $referer     = "http://graphics8.nytimes.com/";

=head1 NAME

scrapepaul.pl - The WTC scrap0r

=head1 DESCRIPTION

I dunno, what do you suggest?

=head2 get_page ( )

This gets the page, it eats the page and then it sets a
referer to the page.  Its all very nice and smells of
potato cakes.  Yum, potato cakes!

=cut

sub get_page {
    my $url = shift;
    my $ua = LWP::UserAgent->new(agent => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)");
    if ($referer) {
    	$ua->default_headers->push_header('referer' => $referer);
    }
    my $resp = $ua->get($url);
    return \$resp->{'_content'};
}

=head2 Get it

Grab the page n stuff

=cut

my $page = ${get_page($url)};

=head2 My testing Mr Robins! :D
$/ = undef;
open FILE, "page.html";
my $page = <FILE>;
close FILE;
=cut

=head2 main

This is where we do the business, I'm gonna get it
as far as I think and then you can make it neat,
maybe add one of those main() routines u like so
much :)

=cut

my $table = ($page =~ m,<table width="768" border="0" cellspacing="0" cellpadding="0">(.*?)</table>,s)[0];
my @rows = ($table =~ m,<tr>(.*?)</tr>,gs);
foreach (@rows) {
    # 0 = name / pdf link
    # 1 = spacer
    # 2 = title
    # 3 = spacer
    # 4 = interview date
    # 5 = spacer
    # 6 = notes
    my @row = ($_ =~ m,<td.*?>(.*?)</td>,gs);
    if ($#row == 6) {
        my $pdf     = ($row[0] =~ m,href="(.*?)",i)[0];
        my $title   = ($row[2] =~ m,<font.*?>(.*?)</font>,i)[0];
        my $date    = ($row[4] =~ m,<font.*?>(.*?)</font>,i)[0];
        my $note    = ($row[6] =~ m,<font.*?>(.*?)</font>,i)[0];
        print "$pdf\n";
        print "$title\n";
        print "$date\n";
        print "$note\n";
        print "\n\n";
    }
}

#print $table;

=head1 TODO

Paul needs to finish this off :)

=head1 AUTHOR

Paul Williams, cpan AT cougasoft DOT org
Paul Robins, paul AT gamingmp DOT com

=head1 COPYRIGHT

This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself.

=cut

1;