rendered paste body<?php
require '/usr/share/pear/HTTP/Request.php';
$start = 'http://www.quepasa.com';
$dbCon = mysql_connect("localhost","root","vxd3sd45") or die(mysql_error());
mysql_select_db("spider");
$options = array('allowRedirects' => true, 'maxRedirects' => 5);
$r = new HTTP_Request($start,$options);
$r->setMethod(HTTP_REQUEST_METHOD_GET);
$r->sendRequest();
$page = $r->getResponseBody();
$query = mysql_query("INSERT INTO urls (url_base,url_title,url_description,url_keywords,url_domain,url_crawled,url_added) VALUES('".$start."','".format_data(getTitle($page))."','".format_data(getDescription($page))."','".format_data(getKeywords($page))."','".format_data(strip_domain($url))."','Yes',now())") or die(mysql_error());
preg_match_all ("/\s+href\s*=\s*[\"\']?([^\s\"\']+)[\"\'\s]+/ims",$page,$matches);
$url_array = $matches[0];
$url_count = count($url_array);
for ($i=0;$i<$url_count;$i++) {
if (preg_match('/http:\/\/(.*?)/i',$url_array[$i])) {
$url = str_replace('href="',"",$url_array[$i]);
$url = str_replace('"',"",$url);
$url = trim($url);
$url = strtolower($url);
if (preg_match('/href=/i',$url)) {
$url = str_replace('href=',"",$url);
}
$query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error());
} elseif (!preg_match('/mailto:(.*?)/i',$urlArray[$i])) {
$url = str_replace('href="',"",$urlArray[$i]);
$url = str_replace('"',"",$url);
$url = trim(strtolower($url));
$current_URL = preg_replace("/([^\/])\/[^\/]+$/i", "\\1", $start);
$url = $start . "/" . $url;
$url = str_replace("/./", "/", $url);
$url = preg_replace("/\/[\/]+/i", "/", $url);
$url = str_replace("http:/", "http://", $url);
$url = str_replace("&", "&", $url);
$query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error());
}
}
function format_data($string) {
return str_replace("'","\'",$string);
}
function strip_domain($url) {
$pattern = '/^(http:\/\/)?([^\/]+)/i';
preg_match($pattern,$url,$domain);
preg_match("/[^\.\/]+\.[^\.\/]+$/", $domain[2],$domain);
return $domain[0];
}
function getTitle($page) {
$pgTitle = preg_match('/<title>(.*?)<\/title>/ims', $page, $title);
return $title[1];
}
function getKeywords($page) {
preg_match('/<meta name="keywords" content="(.*?)">/ims', $page, $keywords);
return $keywords[1];
}
function getDescription($page) {
preg_match('/<meta name="description" content="(.*?)">/ims', $page, $description);
return $description[1];
}
?>