rendered paste body<?php //set varibles $no_tags = array(); //Cuts out the hmtl and php tags $no_lines = array(); // breaks the line aparts $docid_hash = array(); //Document lookup table $word_hash = array(); //List of words hash $tfcount = array(); //Number of documents per word $number_of_documents_containing_word = array(); //number of documents containing given word $tfquery_spaced_words = array(); //counts the number of documents a word appears in $Okapi_score_query = array(); //Okapi score query //only change theses // $division = 0; $unique_in_collection = 220685; $total_number_of_terms = 41872288; //number of terms in the corpus $total_number_of_documents = 84678; //number of documents in the corpus //http://fiji4.ccs.neu.edu/~zerg/lemurcgi_IRclass/lemur.cgi?d=? //get command-line opts $opts = getopt('m:'); if (!$opts) { printUsage(); exit; } $method = empty($opts['m']) ? '' : $opts['m']; $doc_avg = //Calculates the avg number of terms per doc $total_number_of_terms / $total_number_of_documents; //Gets the Document Mapping File $doc_list = file_get_contents('doclist'); //Explodes the file at the Line $doc_list_lines = explode("\n", $doc_list); //Splits at the spaces $doc_list_array[] = array(); foreach($doc_list_lines as $value){ $doc_list_array[] = preg_split("/\s+/", $value); } //Stores in hash with docid as key $doc_list_hash[] = array(); foreach($doc_list_array as $value){ //echo $value[0]; $doc_list_hash[$value[0]] = $value[1]; } //Gets the Query $text = file_get_contents('desc.51-100.short2'); //$text = file_get_contents('two'); $query = array(); //seporates the words using spaces $spaced_words = explode("\n", $text); foreach($spaced_words as $key => $value){ $pattern = "/(^\d*)/"; if (preg_match_all($pattern, $value, $matches_array)){ $pattern = "/D.*$/"; if (preg_match_all($pattern, $value, $matches_array2)){ $query[$matches_array[0][0]]=explode(" ", trim(str_replace('.','', str_replace(',', '', $matches_array2[0][0])))); } } }//$query = array(57 => array("how", "MCI", "has", "been" , "doing", "since", "the", "Bell", "System", "breakup")); // gets the list fro the server foreach($query as $querynumber => $individual_query){ //Gets the inverted list of the $spaced_words foreach($individual_query as $word){ /** * Initialize the cURL session */ $ch = curl_init(); /** * Set the URL of the page or file to download. */ curl_setopt($ch, CURLOPT_URL, 'http://fiji4.ccs.neu.edu/~zerg/lemurcgi_IRclass/lemur.cgi?g=p&d=' . $division . "&v=" . $word); /** * Ask cURL to return the contents in a variable * instead of simply echoing them to the browser. */ curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); /** * Execute the cURL session */ $contents = curl_exec ($ch); /** * Close cURL session */ curl_close ($ch); // var_dump($contents); //cut html tags and php tags $no_tags[$word] = strip_tags($contents); } //places the inverted list into $word_hash foreach ($no_tags as $word => $inverted_table){ $pattern = "/\s([0-9]+)/"; $matches_array = array(); $lines = explode("\n", $inverted_table); //breaks up lines //skips the first 7 lines and just goes to the invertered list for($count = 7; $count < count($lines); $count++) { if (preg_match_all($pattern, $lines[$count], $matches_array)){ $docid = $matches_array[1][0]; $doclen = $matches_array[1][1]; $tf = $matches_array[1][2]; $cf_dfline = $lines[5]; if (preg_match_all($pattern, $cf_dfline, $matches_array)){ $cft = $matches_array[0][0]; $df = $matches_array[1][1]; $docid_hash[$docid] = array('doc_len' => $doclen, 'tf' => $tf, 'ctf' => $cft, 'df' => $df); } } } $word_hash[$word] = $docid_hash; $number_of_documents_containing_word[$word] = count($word_hash[$word]); } // Computes the scores foreach ($word_hash as $word => $value){ $myscore = array(); // $ml_score = array();// $jm_score = array();// $Okapi_score = array();// $idf_tf =array(); $word_total= 0; $cf= 0; foreach($value as $docid => $value2){ // gets the number of documents containing the word //echo $value2['doc_len'] . " \n"; $accumulater = $value2['tf'] / ($value2['tf'] + .5 + 1.5 * ($value2['doc_len'] / $doc_avg)); if ($method = 'okapi') { $myscore[$docid] = $accumulater + $myscore[$docid]; //echo $querynumber . " " . "Q0 " . $doc_list_hash[$docid] . " 0 " . $Okapi_score[$docid] . " okapi" . "\n" ; $Okapi_score_matrix= array(); foreach($myscore as $key => $score){ $Okapi_score_matrix[] = array($score, $key); } sort($Okapi_score_matrix); //var_dump($Okapi_score_matrix); for($x = (count($Okapi_score_matrix)-1); $x > (count($Okapi_score_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$Okapi_score_matrix[$x][1]] . " 0 " . $Okapi_score_matrix[$x][0] . " okapi" . "\n" ; // file_put_contents("results" . $division . "/okapi.txt", $temp, FILE_APPEND); echo $temp; } } elseif ($method = 'tf_idf') { //problem 2 $myscore[$docid] = $myscore[$docid] + ($accumulater * log10($total_number_of_documents / $value2['df'])); //echo $querynumber . " " . "Q0 " . $doc_list_hash[$docid] . " 0 " . $idf_tf[$docid] . " idf_score" . "\n" ; $idf_tf_matrix= array(); foreach($myscore as $key => $score){ $idf_tf_matrix[] = array($score, $key); } sort($idf_tf_matrix); for($x = (count($idf_tf_matrix)-1) ; $x > (count($idf_tf_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$idf_tf_matrix[$x][1]] . " 0 " . $idf_tf_matrix[$x][0] . " idf_score" . "\n" ; // file_put_contents("results" . $division . "/idf_score.txt", $temp, FILE_APPEND); echo $temp; } } elseif ($method = 'ml') { //problem 3 $ml_accumulator= ($value2['tf']+1) / ($value2['doc_len'] + $unique_in_collection); $myscore[$docid]= $myscore[$docid] + $ml_accumulator; //echo $querynumber . " " . "Q0 " . $doc_list_hash[$docid] . " 0 " . $ml_score[$docid] . " ML_score" . "\n" ; $ml_score_matrix= array(); foreach($myscore as $key => $score){ $ml_score_matrix[] = array($score, $key); } sort($ml_score_matrix); for($x = (count($ml_score_matrix)-1); $x > (count($ml_score_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$ml_score_matrix[$x][1]] . " 0 " . $ml_score_matrix[$x][0] . " ml_score" . "\n" ; // file_put_contents("results" . $division . "/ml_score.txt", $temp, FILE_APPEND); echo $temp; } } elseif ($method = 'jm') { //prob 4 $myscore[$docid] = $myscore[$docid] + ((($value2['tf']/$value2['doc_len']) *.8) + (2* ($value2['ctf'] / $total_number_of_terms))); /* $value2['tf'] is wrong suppose to be cf */ //echo $querynumber . " " . "Q0 " . $doc_list_hash[$docid] . " 0 " . $jm_score[$docid] . " jm_score" . "\n" ; //number of times the word appears in the documents /numer owrkds in doc $jm_score_matrix= array(); foreach($myscore as $key => $score){ $jm_score_matrix[] = array($score, $key); } sort($jm_score_matrix); for($x = (count($jm_score_matrix)-1); $x > (count($jm_score_matrix)-1001); $x--){ //echo $querynumber . " " . "Q0 " . $doc_list_hash[$jm_score_matrix[$x][1]] . " 0 " . $jm_score_matrix[$x][0] . " jf_score" . "\n" ; $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$jm_score_matrix[$x][1]] . " 0 " . $jm_score_matrix[$x][0] . " jf_score" . "\n" ; // file_put_contents("results" . $division ."/jm_score.txt", $temp, FILE_APPEND); echo $temp; } } else {die;} } } /* $Okapi_score_matrix= array(); foreach($Okapi_score as $key => $score){ $Okapi_score_matrix[] = array($score, $key); } sort($Okapi_score_matrix); //var_dump($Okapi_score_matrix); for($x = (count($Okapi_score_matrix)-1); $x > (count($Okapi_score_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$Okapi_score_matrix[$x][1]] . " 0 " . $Okapi_score_matrix[$x][0] . " okapi" . "\n" ; file_put_contents("results" . $division . "/okapi.txt", $temp, FILE_APPEND); } unset($Okapi_score); $idf_tf_matrix= array(); foreach($idf_tf as $key => $score){ $idf_tf_matrix[] = array($score, $key); } sort($idf_tf_matrix); for($x = (count($idf_tf_matrix)-1) ; $x > (count($idf_tf_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$idf_tf_matrix[$x][1]] . " 0 " . $idf_tf_matrix[$x][0] . " idf_score" . "\n" ; file_put_contents("results" . $division . "/idf_score.txt", $temp, FILE_APPEND); } $ml_score_matrix= array(); foreach($ml_score as $key => $score){ $ml_score_matrix[] = array($score, $key); } sort($ml_score_matrix); for($x = (count($ml_score_matrix)-1); $x > (count($ml_score_matrix)-1001); $x--){ $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$ml_score_matrix[$x][1]] . " 0 " . $ml_score_matrix[$x][0] . " ml_score" . "\n" ; file_put_contents("results" . $division . "/ml_score.txt", $temp, FILE_APPEND); } unset($ml_score); $jm_score_matrix= array(); foreach($idf_tf as $key => $score){ $jm_score_matrix[] = array($score, $key); } sort($jm_score_matrix); for($x = (count($jm_score_matrix)-1); $x > (count($jm_score_matrix)-1001); $x--){ //echo $querynumber . " " . "Q0 " . $doc_list_hash[$jm_score_matrix[$x][1]] . " 0 " . $jm_score_matrix[$x][0] . " jf_score" . "\n" ; $temp = $querynumber . " " . "Q0 " . $doc_list_hash[$jm_score_matrix[$x][1]] . " 0 " . $jm_score_matrix[$x][0] . " jf_score" . "\n" ; file_put_contents("results" . $division ."/jm_score.txt", $temp, FILE_APPEND); } unset($jm_score, $idf_tf); */ }?>