All pastes #276896 Raw Edit

Mine

public text v1 · immutable
·published 2006-12-12 19:53 UTC
rendered paste body
<?php

	require '/usr/share/pear/HTTP/Request.php';
	$start = 'http://www.quepasa.com';

	$dbCon = mysql_connect("localhost","root","vxd3sd45") or die(mysql_error());
	mysql_select_db("spider");

	$options = array('allowRedirects' => true, 'maxRedirects' => 5);
	$r = new HTTP_Request($start,$options);
	$r->setMethod(HTTP_REQUEST_METHOD_GET);
	$r->sendRequest();
	$page = $r->getResponseBody();

	$query = mysql_query("INSERT INTO urls (url_base,url_title,url_description,url_keywords,url_domain,url_crawled,url_added) VALUES('".$start."','".format_data(getTitle($page))."','".format_data(getDescription($page))."','".format_data(getKeywords($page))."','".format_data(strip_domain($url))."','Yes',now())") or die(mysql_error());

	preg_match_all ("/\s+href\s*=\s*[\"\']?([^\s\"\']+)[\"\'\s]+/ims",$page,$matches);
	$url_array = $matches[0];
	$url_count = count($url_array);

	for ($i=0;$i<$url_count;$i++) {
		if (preg_match('/http:\/\/(.*?)/i',$url_array[$i])) {
			$url = str_replace('href="',"",$url_array[$i]);
			$url = str_replace('"',"",$url);
			$url = trim($url);
			$url = strtolower($url);

			if (preg_match('/href=/i',$url)) {
				$url = str_replace('href=',"",$url);
			}

			$query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error());
		} elseif (!preg_match('/mailto:(.*?)/i',$urlArray[$i])) {
			$url = str_replace('href="',"",$urlArray[$i]);
			$url = str_replace('"',"",$url);
			$url = trim(strtolower($url));
		
			$current_URL = preg_replace("/([^\/])\/[^\/]+$/i", "\\1", $start);
			$url = $start . "/" . $url;
			$url = str_replace("/./", "/", $url);
			$url = preg_replace("/\/[\/]+/i", "/", $url);
			$url = str_replace("http:/", "http://", $url);
			$url = str_replace("&amp;", "&", $url);

			$query = mysql_query("INSERT INTO urls (url_base,url_domain,url_crawled,url_added) VALUES('".format_data($url)."','".format_data(strip_domain($url))."','No',now())") or die(mysql_error());
		}
	}

	function format_data($string) {
		return str_replace("'","\'",$string);
	}

	function strip_domain($url) {
		$pattern = '/^(http:\/\/)?([^\/]+)/i';
		preg_match($pattern,$url,$domain);
		preg_match("/[^\.\/]+\.[^\.\/]+$/", $domain[2],$domain);
		return $domain[0];
	}

	function getTitle($page) {
		$pgTitle = preg_match('/<title>(.*?)<\/title>/ims', $page, $title);
		return $title[1];
	}

	function getKeywords($page) {
		preg_match('/<meta name="keywords" content="(.*?)">/ims', $page, $keywords);
		return $keywords[1];
	}

	function getDescription($page) {
		preg_match('/<meta name="description" content="(.*?)">/ims', $page, $description);
		return $description[1];
	}

?>