All pastes #357968 Raw Edit

scraper

public php v1 · immutable
·published 2007-02-16 00:30 UTC
rendered paste body
<?php$keywords = $pagekeyword;require_once("ses-config.php");require_once("ses.php");require_once("pear.php");if(!$keywords && !$keyword){  $keywords = $mainkeyword;}if(!$keywords && $keyword){  $keywords = $keyword;}if($mylinkFile){  $linkFile = $mylinkFile;}#################Populate Single Keyword and Write to File###############################if (!function_exists('popSEfly')) {function popSEfly($keywords){	global $min;	global $max;	global $shuffle;	global $source;	global $minparagraphs;	global $maxparagraphs;	global $PathToFile;	global $PrintTitle;	global $results;	global $densityControl;        global $preTitle;         global $postTitle;         global $searticlecache;	global $linkFile;	$current_keywords = "";	$illegal_chars = "";	$filename = "";	$FullPathName = "";	$paragraphs = "";	$articleTitle = "";	$length = "";	$searticle = "";	$searticlefinal = "";	$current_keyword = $keywords;	$current_keyword = trim($current_keyword);	$illegal_chars = array("(", ")", ".", "<", ">", ",", ":", ";", "{", "}", "+", "=", "[", "]", "/");	$filename = strtolower($current_keyword);	$filename = str_replace($illegal_chars, "", $filename);	$filename = str_replace(" ", "-", $filename);	$FullPathName = $PathToFile . $filename . ".html";	$searticlecache = $searticlecache * 86400; // Convert cache expiration to days   	if ( (file_exists($FullPathName)) && (filesize($FullPathName) >= 200) && (time() < (filemtime($FullPathName) + $searticlecache)) ) {     	include($FullPathName);	}else{   	$paragraphs = rand($minparagraphs,$maxparagraphs);	if(!$results){	$results = scrapeSE($current_keyword);	}//Creates searticle folder	if(!is_dir($PathToFile) && is_file($PathToFile))			{trigger_error("Error: cannot make folder <b>" . $PathToFile . "</b> as it exists as a file", E_USER_ERROR);}	if(!is_dir($PathToFile) && !is_file($PathToFile))		{			if(mkdir($PathToFile, 0777) === FALSE)				{trigger_error("Error: cannot make folder <b>" . $PathToFile . "</b> mkdir operation failed", E_USER_ERROR);}			else			{@chmod($PathToFile, 0777);}		}	if(!is_writable($PathToFile))			{@chmod($PathToFile, 0777);}//Creates Paragraphs			$i = 1;	$l = 1;	$five = 0;	$ten = 0;	while ($i <= $paragraphs) {		$i++; 		$searticle = searticle($results,rand($min,$max),$shuffle);		$articleTitle = articletitle($results);		$max = $max+10;		$min_half = ($min/2); 		$max_half = ($max*1.25); 		$searticlefinal_wrd_cnt = str_word_count($searticle);		$setitlefinal_wrd_cnt = str_word_count($articleTitle);/*		echo "<br>SE Article Word count: ($searticlefinal_wrd_cnt)";		echo "<br>SE Title Word count: ($setitlefinal_wrd_cnt)";*/		//Checks to make sure that article scraped correctly		if($searticlefinal_wrd_cnt >= $min_half && $searticlefinal_wrd_cnt < $max_half && $setitlefinal_wrd_cnt >= 1) { //Opens the file for writing in the correct status												if($i == 2)				{				$fp = fopen($FullPathName, 'w');				}				else				{				$fp = fopen($FullPathName, 'a');				}//Writes a Title to keyword file if you want it								if($i == 2 && ($PrintTitle))				{				fwrite($fp,$preTitle.$articleTitle.$postTitle);				}//Writes the paragraph to a keyword file			fwrite($fp, $searticle);			fwrite($fp, "<br>\n<br>\n");			fclose($fp);			}		else			{			$i = ($i - 1);						if($l >= 5 && !$five){//			  echo "<br><b>L over 5</b>";			  $results = scrapeSE($current_keyword);			  if(count($results) > 1){			    $five = "complete";			  }  			}					if($l >= 10  && file_exists($linkFile) && !$ten){//			echo "<br><b>L over 10</b>";						$keywordfile = file($linkFile);			$cnt = count($keywordfile)-1;			$num = rand(0,$cnt);			$alternatewrd = $keywordfile[$num];			$alternatewrd = trim($alternatewrd);			$results = scrapeSE($alternatewrd);						if(count($results) > 1){			  $ten = "complete";			}						}			if($l >= 20){			exit("");			}						$l++;			}		}	if($densityControl){	  require_once("density.php");	  density($keywords,$FullPathName);	}	else{	  include($FullPathName);	  }}}}popSEfly($keywords);?>