grab.pl

#!/usr/local/bin/perl

# Extended UNIX Ripper v10
# Todo: implement -n and -s
#	Add -1 to number images
#	Abstracted wget/curl/LWP
#	Add -t to test which images are down without downloading
#	Add -r to recursively get indexes
#	Add interactive mode
#

#Copyright (c) 2006 grip
#Permission is hereby granted, free of charge, to any person
#obtaining a copy of this software and associated documentation
#files (the "Software"), to deal in the Software without
#restriction, including without limitation the rights to use,
#copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the
#Software is furnished to do so, subject to the following
#conditions:
#
#The above copyright notice and this permission notice shall be
#included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#OTHER DEALINGS IN THE SOFTWARE.

use strict;
use warnings;
use LWP;
use HTTP::Headers;
use HTTP::Cookies::Netscape;
use Digest::MD5 qw(md5_hex);

$| =1; # Flush print every print

sub directimg($$)
{
	my($url, $browser) = @_;
	
	$url =~ /\/([^\/]+)$/ or die "directimg() recieved a bad value: ".$url;
	my $imageName = $1;
	
	if (-e $imageName)
	{
		print STDERR $imageName." already exists.\n";
		return 1;
	}
	
	print STDERR "Getting ".$imageName."\n";
	
	my $response = $browser->get($url);
	if(not $response->is_success)
	{
		print STDERR $imageName." failed: ".$response->status_line."\n";
		return 0;
	}
	if ( not $response->content_type =~ /image/)
	{
		print STDERR $imageName." failed: Image not detected.\n";
		return 0;
	}
	if ( length($response->content) < 50000)
	{
		print STDERR $imageName." failed: Below minimum file size.\n";
		return 1;
	}
	my $IMAGE;
	if (not open($IMAGE, ">".$imageName))
	{
		print STDERR "Couldn't open ".$imageName." for writing.\n";
		return 0;
	}
	print $IMAGE $response->content;
	close($IMAGE);
	return 1;
}
sub rapidshare($$)
{
	my($url, $browser) = @_;
	
	$url =~ /http:\/\/(?:www\.)?rapidshare.de(\/files\/[0-9]+)\/(.*)$/ or die "rapidshare() recieved a bad value: ".$url;
	my $uri = $1."/".$2;
	my $filename = $2;
	
	print STDERR "Getting the Rapidshare URL of ".$filename."\n";
	
	$browser->cookie_jar( HTTP::Cookies::Netscape->new('file'=>'cookies.txt', 'autosave'=>1) );
	$browser->default_headers->push_header('Referer', $url);
	my $response = $browser->post('http://www.rapidshare.de/', 
				[ 'uri'=>$uri, 'dl.start'=>'PREMIUM' ]);
	if(not $response->is_success)
	{
		print STDERR $filename." failed: ".$response->status_line."\n";
		return 0;
	}
	if ( not $response->content =~ /<a href="(http:\/\/[a-z0-9]+\.rapidshare\.de\/files\/[0-9]+\/[0-9]+\/[^"]+)">/s )
	{
		print STDERR $filename." failed: Couldn't find url in requested page.\n";
		return 0;
	}
	my $rapidshareUrl = $1;
	my $RAPIDSHARE;
	if (not open($RAPIDSHARE, ">>rapidshare.txt"))
	{
		print STDERR "Couldn't open rapidshare.txt for writing.\n";
		return 0;
	}
	print $RAPIDSHARE $rapidshareUrl."\n";
	close($RAPIDSHARE);
	return 1;
}
sub dumparump($$$)
{
	my($id, $browser, $number) = @_;
	my $url = 'http://www.dumparump.com/view.php?id='.$id;
	
	if(-e $number.'.jpg')
	{
		print STDERR $number.'.jpg exists already.\n';
		return 1;
	}
	
	$browser->default_headers->push_header('Referer', 'http://www.google.com/?s='.$id);
	my $response = $browser->get($url);
	$browser->default_headers->remove_header('Referer');
	if( not $response->is_success)
	{
		print STDERR $url." failed : ".$response->status_line.".\n";
		return 0;
	}
	
	if(not $response->content =~ /<img src="http:\/\/www.dumparump.com\/image\/([0-9]+)\//)
	{
		print STDERR "Couldn't find the random number for ".$id."\n";
		return 0;
	}
	my $randNo = $1;
	
	$browser->default_headers->push_header('Referer', $url);
	$response = $browser->get('http://www.dumparump.com/image/'.$randNo.'/'.$id.'.jpg');
	$browser->default_headers->remove_header('Referer');
	if (not ( $response->is_success and $response->content_type =~ /image/ ) )
	{
		print STDERR "Couldn't get ".'http://www.dumparump.com/image/'.$randNo.'/'.$id.'.jpg'." : ".$response->status_line."\n";
		return 0;
	}
	
	my $IMAGE;
	if (not open($IMAGE, ">".$number.'.jpg'))
	{
		print STDERR "Couldn't open ".$number.'.jpg'." for writing.\n";
		return 0;
	}
	print $IMAGE $response->content;
	close($IMAGE);
	return 1;
}

sub imagevenue($$)
{
	my($url, $browser) = @_;
	
	$url =~ /http:\/\/([^\.]*)\./ or die "imagevenue() recieved bad value: ".$url;
	my $server = $1;
	$url =~ /image=(.*)/ or die "imagevenue() recieved bad value: ".$url;
	my $imageName = $1;
	
	$imageName =~ /^(?:[a-f0-9]+_)?(.+?)(?:(?:_[0-9]{1,3})?_[0-9]{1,4}lo)?(\.(?:jpe?g)|(?:png))$/i
		or die "imagevenue() recieved bad value: ".$imageName;
	my $finalImageName = $1.$2;
	
	if (-e $finalImageName)
	{
		print STDERR $finalImageName." already exists.\n";
		return 1;
	}
	
	print STDERR "Getting ".$finalImageName."\n";
	
	
	my $response = $browser->get($url);
	if( not $response->is_success)
	{
		if($response->status_line =~ /302/)
		{
			print STDERR $finalImageName." failed : Deleted\n";
			return 1;
		} else {
			print STDERR $finalImageName." failed : ".$response->status_line.".\n";
			return 0;
		}
	}
	
	if(not $response->content =~ /SRC="([^\/]+)\/([^\/]+)/)
	{
		print STDERR "Couldn't find the random string for ".$finalImageName."\n";
		return 0;
	}
	my $randString = $1;
	my $loc = $2;
	
	$browser->default_headers->push_header('Referer', $url);
	$response = $browser->get('http://'.$server.'.imagevenue.com/'.$randString.'/'.$loc.'/'.$imageName);
	$browser->default_headers->remove_header('Referer');
	if (not ( $response->is_success and $response->content_type =~ /image/ ) )
	{
		print STDERR "Couldn't get ".$finalImageName." : ".$response->status_line."\n";
		return 0;
	}
	
	my $IMAGE;
	if (not open($IMAGE, ">".$finalImageName))
	{
		print STDERR "Couldn't open ".$finalImageName." for writing.\n";
		return 0;
	}
	print $IMAGE $response->content;
	close($IMAGE);
	return 1;
}
sub shareapic ($$) {
	my($url, $browser) = @_;
	
	$url =~ /http:\/\/www.shareapic.net\/content\.php\?id=([0-9]+)/
		or die "shareapic() recieved bad value: ".$url;
	my $id = $1;
	my $imageName = "000000000";
	my $start = length($imageName) - length($id);
	substr($imageName, $start) = $id;
	$imageName = $imageName . ".jpg";
	
}

# Create the browser
my $ripLocation = 'http://rip.to/';

my $browser = LWP::UserAgent->new(
	'default_headers' => new HTTP::Headers('User-Agent' => 'Mozilla/4.76 [en] (Win98; U)',
		'Accept' => 'image/gif, image/x-xbitmap, image/jpeg, 
			image/pjpeg, image/png, */*',
		'Accept-Charset' => 'iso-8859-1,*,utf-8',
		'Accept-Language' => 'en-US'));
$browser->max_redirect(0);
$browser->cookie_jar( HTTP::Cookies::Netscape->new('file'=>'cookies.txt', 'autosave'=>1) );

print STDERR "Extended UNIX ripper (v10)\n\n";

sub usage {
	print STDERR "Usage:\n";
	print STDERR "\tgrab.pl\n";
	print STDERR "\tgrab.pl -t=<topicid>|-p=<postid> [-f=folder] [-o] [-n=<number> -s=<number>] [-u=<url>]\n";
	print STDERR "\tgrab.pl -i [-f=folder] [-n=<number> -s=<number>] [-u=<url>]\n\n";
	print STDERR "Arguments:\n";
	print STDERR "\tWithout arguments grab.pl runs in interactive mode.\n";
	print STDERR "\t-t,-p :\tRetrieve HTML from a topic/post in a vBulletin forum.\n";
	print STDERR "\t-i :\tInput parseable HTML from STDIN.\n";
	print STDERR "\t-o :\tOutput parseable HTML from forum without parseing. Makes \n";
	print STDERR "\t\t-f, -n, and -s ignored.\n";
	print STDERR "\t-f :\tSave to this folder. If using a topic/post id and this\n";
	print STDERR "\t\tis not given the name of the thread will be used.\n";
	print STDERR "\t\tIf using -i and this is not given the images will be saved\n";
	print STDERR "\t\tto the current folder.\n";
	print STDERR "\t-n,-s :\tOnly get every -n=<number>th image, starting at image\n";
	print STDERR "\t\tnumber -s=<number>. This allows you to split a large thread\n";
	print STDERR "\t\tamongst multiple instances of grab.pl .\n";
	print STDERR "\t-u :\tAdd a URL to the start of all directly or indirectly linked\n";
	print STDERR "\t\timages.\n\n";
	print STDERR "Examples:\n";
	print STDERR "\tgrab.pl -t=98283\n\t\tGet topic id 98283.\n";
	print STDERR "\tcat images.html | grab.pl -i -f=asdf\n";
	print STDERR "\t\tGet HTML from images.html, put into a folder called asdf.\n";
	print STDERR "\tgrab.pl -t=98283 -n=2 -s=1 & grab.pl -t=98283 -n=2 -s=2 &\n";
	print STDERR "\t\tSplit topic amongst two grab.pls simultaneously for increased speed.\n";
	print STDERR "\tgrab.pl -t=98283 -o > HTML.html && for i in 1 2 3 4 5; do \\\n";
	print STDERR "\tcat HTML.html | grab.pl -i -f=MyFolder -n=5 -s=".'"$i"'." 2>&1 \\\n";
	print STDERR "\t| sed -e 's/^/'".'"$i"'."': /' & done\n";
	print STDERR "\t\tUse a grab.pl to download a thread, then start 5 grab.pl to\n";
	print STDERR "\t\tdownload the contents.\n\n";
}

my $text="";
my $folder=".";
if(grep(/^-i$/, @ARGV) == 1)
{
	#We're getting text from stdin
	print STDERR "Getting text from STDIN.\n\n";
	
	while (<STDIN>)
	{
		$text = $text.$_;
	}
	
} elsif(grep(/(?:-t=)|(?:-p=)[0-9]+$/, @ARGV) == 1) {
	
	my $id = (grep /(?:-t=)|(?:-p=)[0-9]+$/, @ARGV)[0];
	$id =~ s/^-//;
	
	# We're downloading from RiP
	sub isLoggedInToRip()
	{
		my $COOKIES;
		open($COOKIES, '<cookies.txt');
		my $cookies = join('', <$COOKIES>);
		close($COOKIES);
		return ($cookies =~ /rip\.to.*rippassword/);
	}

	while (not isLoggedInToRip)
	{
		print STDERR "You're not currently logged in.\nPlease enter your username:\n> ";
		my $username = <STDIN>;
		chomp($username);
		print STDERR "Please enter your password:\n> ";
		my $password = <STDIN>;
		chomp($password);
		$password = md5_hex $password;
		
		my $response = $browser->post($ripLocation.'login.php', 
				[
				'do'=>'login', 'forceredirect'=>'1','url'=>'/',
				'vb_login_md5password'=>$password,'vb_login_username'=>$username,
				'cookieuser'=>'1'
				]);
		
		$response->is_success or do die "Couldn't log on : ".$response->status_line.".\n";
		$browser->cookie_jar->save;
		print STDERR "\n";
	}

	print STDERR "Getting " . $id . "..\n";

	my $response;
	if ( $id =~ /p/ )
	{
		$response = $browser->get($ripLocation . 'showpost.php?' . $id );
	}
	else
	{
		$response = $browser->get($ripLocation . 'showthread.php?' . $id );
	}
	
	$response->is_success or die "Couldn't download ".$id." : ".$response->status_line;
	my $input = $response->content;
	
	if ( $id =~ /t/ )
	{
		$input =~ /Search this Thread/ or die "Thread doesn't seem to be valid";
	}

	$input =~ /<title>RiP - View Single Post - ([^<]*?)<\/title>/ or $input =~ /<title>([^<]*?)(?: - Page [0-9]+)? - RiP<\/title>/ or die "Thread doesn't seem to be valid";

	$folder = $1;
	$folder =~ s/[^A-Za-z0-9_-]/_/g;

	# Get any extra pages needed
	if($input =~ /Page 1 of ([0-9]+)/)
	{
		my $pageNo=$1;
		print STDERR "There are ".$pageNo." pages in this thread.\n";
		print STDERR "Retrieving 1";
		for(my $i=2; $i<=$pageNo; $i++)
		{
			my $response = $browser->get($ripLocation . 'showthread.php?' . $id .'&page='.$i );
			$response->is_success or die "Couldn't download ".$id." page ".$i." : ".$response->status_line.".\n";
			$input = $input.$response->content;
			print STDERR ", " .$i;
		}
		print STDERR "\n";
	}
	print STDERR "Successfully got ".$id.". Title is '".$folder."'\n\n";

	# Filter out message text
	print STDERR "Filtering message text from vBulletin HTML.\n\n";
	foreach ($input =~ /<!-- message -->.*?<div[^>]+>(.*?)<\/div>.*?<!-- \/ message -->/sg)
	{
		$text .= $_;
	}
	$input="";
	
	if(grep(/^-o/, @ARGV) == 1) {
		print $text;
		print STDERR "Finished outputting vBulletin message HTML.\n";
		exit;
	}
} elsif (grep (/-h/, @ARGV) == 1) {
	usage;
} else {
	usage;
	print STDERR "Invalid arguments given, refer to usage details.\n";
	exit;
}

if (grep(/^-f=/, @ARGV) == 1) {
	$folder = (grep /^-f=/, @ARGV)[0];
	$folder =~ s/^-f=//;
	print STDERR "Using folder name ".$folder.".\n";
}

my $preurl = "";
if (grep(/^-u=/, @ARGV) == 1) {
	$preurl = (grep /^-u=/, @ARGV)[0];
	$preurl =~ s/^-u=//;
	print STDERR "Pre-appending ".$preurl." to directly linked images.\n";
}

if ( not -e $folder ) {
	print STDERR $folder." does not exist. Creating it.\n";
	mkdir $folder or die "Could not successfully create ".$folder;
} elsif ( not -d $folder ) {
	die $folder." exists, but isn't a folder.";
}
chdir $folder or die "Could not successfully change directory to ".$folder;
print STDERR "\n";

# Begin adding urls to @urls
my @urls;
print STDERR "Extracting URLs from input.\n";
# ImageVenue
print STDERR "Extracting imagevenue URLs.. ";
foreach ($text =~ m@<a href="(http://img[0-9]+\.imagevenue.com/img.php\?(?:loc=loc[0-9]+&amp;)?image=[^"]+)"@sg)
{
	my $url = "I-".$_;
	$url =~ s/&amp;/&/g;
	
	grep(/$url/, @urls) == 0 and push(@urls, $url);
	
	@urls % 100 == 0 and print STDERR @urls.".. ";
}
print STDERR "\n";
# Direct
print STDERR "Extracting directly linked image URLs.. ";
foreach ($text =~ /<img[^>]*(?:(?:src)|(?:href))="([^?"]+\.(?:jpe?g)|(?:png))"[^>]*>(?!<\/a>)/isg)
{
	my $url = $_;
	
	if ( $preurl =~ /^http:/ )
	{
		$url = $preurl . $url;
	}
	
	$url = "D-".$url;
	
	grep(/$url/, @urls) == 0 and push(@urls, $url);
	
	@urls % 100 == 0 and print STDERR @urls.".. ";
}
print STDERR "\n";
# Thumbnail
print STDERR "Extracting thumbnail/non-directly linked image URLs.. ";
foreach ($text =~ /<a[^>]*(?:(?:src)|(?:href))="([^?"]+\.(?:jpe?g)|(?:png))"[^>]*>/isg)
{
	my $url = $_;
	
	if ( $preurl =~ /^http:/ )
	{
		$url = $preurl . $url;
	}
	
	$url = "D-".$url;
	
	grep(/$url/, @urls) == 0 and push(@urls, $url);
	
	@urls % 100 == 0 and print STDERR @urls.".. ";
}
print STDERR "\n";

print STDERR "Extracting dumparump URLs.. ";
foreach ($text =~ /<a[^>]*(?:href)="http:\/\/www.dumparump.com\/view.php\?id=([^"]+)"/isg)
{
	my $url = $_;
	
	$url = "DaR-".$url;
	
	grep(/$url/, @urls) == 0 and push(@urls, $url);
	
	@urls % 100 == 0 and print STDERR @urls.".. ";
}
print STDERR "\n";

#Rapidshare
print STDERR "Extracting rapidshare URLs.. ";
foreach ($text =~ /(rapidshare.de\/files\/[0-9]+\/[^\.]*?\.[a-z0-9]{2,3})[^a-zA-Z0-9\._-]/sg)
{
	my $url = "R-http://".$_;
	
	grep(/$url/, @urls) == 0 and push(@urls, $url);
	
	@urls % 100 == 0 and print STDERR @urls.".. ";
}
print STDERR "\n";
sub isLoggedInToRapidshare()
{
	my $COOKIES;
	open($COOKIES, '<cookies.txt');
	my $cookies = join('', <$COOKIES>);
	close($COOKIES);
	return ($cookies =~ /rapidshare\.de.*user/);
}

if(grep(/^R-/, @urls) != 0 and not isLoggedInToRapidshare)
{
	@urls = grep(/^[^R]-/, @urls);
}

$text = ""; # No longer needed


#Filtering complete
@urls == 0 and die "No URLs to download.";

print STDERR "\n";

# If we're only doing some of the urls, seperate them out now.
my($start, $step);
$start = 0;
$step = 0;

if(grep(/^-s=[1-9][0-9]*$/, @ARGV) == 1 and grep(/^-n=[1-9][0-9]*$/, @ARGV) == 1)
{	
	$start = (grep /^-s=[1-9][0-9]*$/, @ARGV)[0];
	$start =~ s/-s=//;
	
	$step = (grep /^-n=[1-9][0-9]*$/, @ARGV)[0];
	$step =~ s/-n=//;
	
	print STDERR "Seperating designated portion of URLs (s=".$start.",n=".$step.").\n\n";
 	
	# This could be done more efficiently with splice
	my @dividedurls;
	for (my $i = ($start-1); $i < @urls; $i = $i + $step)
	{
		push(@dividedurls, $urls[$i]);
	}
	@urls = @dividedurls;
}

my $tries = 1;
while ($tries <= 3 and @urls > 0)
{
	my $urlNo = @urls;
	print STDERR "Attempting to get ".$urlNo." urls".($tries>1 ? ", attempt ".$tries : "").".\n";
	print STDERR "Current/Remaining/Original\n";
	# Start from the top, so we can remove them
	for (my $i=(@urls-1); $i>=0; $i--)
	{
		my $url = $urls[$i];
		print STDERR ($i+1)."/".@urls."/".$urlNo.": ";
		
		print $url."\n";
		my $success;
		
		for ($url)
		{
			/^I-/ and do
			{
				$url =~ s/^I-//;
				$success = imagevenue($url, $browser->clone);
				last;
			};
			
			/^R-/ and do
			{
				$url =~ s/^R-//;
				$success = rapidshare($url, $browser->clone);
				last;
			};
			
			/^D-/ and do
			{
				$url =~ s/^D-//;
				$success = directimg($url, $browser->clone);
				last;
			};
			
			/^DaR-/ and do
			{
				$url =~ s/^DaR-//;
				
				my $num;
				if ( $start != 0 and $step != 0 )
				{
					$num = $i * $step + $start;
				}
				else
				{
					$num = $i + 1;
				}
				$success = dumparump($url, $browser->clone, $num);
				last;
			};
			
			die $url." is not a valid url.";
		}
		if($success)
		{
			splice(@urls,$i,1);
		}
	}
	$tries++;
}

unlink "cookies.txt" or die "Could not remove cookies.txt: $!";

print STDERR "Finished downloading.\n";

if(@urls != 0) {
	my $FAILED;
	open($FAILED, ">>FailedDownloads.".$$.".txt") or die "Could not successfully open FailedDownloads.txt";
	print $FAILED join("\n", @urls);
	close($FAILED);
	print STDERR "There are ".@urls." failed downloads, which have been written to ".$folder."/FailedDownloads.".$$.".txt .\n";
} else {
	print "All downloads completed successfully.\n";
}