<?php/* Takes N text files as lists, checks every one line by line and echoes exactly one of each line to outX.txt, X being a number, as well as all the repeated lines to errX.txt Try running this against /usr/share/dict */$files = array();switch($argv[1]) { case "-d": if(is_dir($argv[2])) { echo date("[Y-m-d h:i:s]")." Gathering files from ".$argv[2]."\n"; foreach(glob($argv[2]."/*") as $globread) { if(is_file($globread)) { $files[] = $globread; } } } else { $error = 3; } break; case NULL: case "-h": echo "Syntax: php deletemultiples.php [-d directory|file1 [fileN]|-h]\n"; die(); break; default: if($argc > 1) { unset($argv[0]); $files = array_values($argv); } else { $error = 2; } break;}foreach($files as $file) { if(!file_exists($file)) { $error = 1; }}switch($error) { case 1: die("FATAL: One or more of the files you specified do not exist\n"); break; case 2: die("FATAL: You didn't specify at least one file to process\n"); break; case 3: die("FATAL: The argument given is not a directory\n"); break; default: echo date("[Y-m-d h:i:s]")." Everything OK. Starting check...\n"; break;}$filecounter = 1;while(!$out) { if (!file_exists("out".$filecounter.".txt")) { $out = fopen("out".$filecounter.".txt", "a"); } else { $filecounter++; }}$filecounter = 1;while(!$err) { if (!file_exists("err".$filecounter.".txt")) { $err = fopen("err".$filecounter.".txt", "a"); } else { $filecounter++; }}$alreadycopied = array();foreach($files as $file) { $inprocess = fopen($file, "r"); while(!feof($inprocess)) { $line = fgets($inprocess); if(!isset($alreadycopied[str_replace("\n", "", $line)])) { fwrite($out, str_replace("\n", "", $line)."\n"); $alreadycopied[str_replace("\n", "", $line)] = 'Y'; } else { fwrite($err, str_replace("\n", "", $line)."\n"); $conflicts++; } } fclose($inprocess); echo date("[Y-m-d h:i:s]")." Finished file ".$file."\n";}echo sprintf(date("[Y-m-d h:i:s]")." Finished. Copied %s individual lines. Found %s duplicates.\n", sizeof($alreadycopied), $conflicts);?>