sub get_text {
my @a = split /\./,"$_[0]";
$type = $filetypes{"$a[1]"};
unless ($type) {
$type = 'null';
}
#my $target = "$_[1].$type";
#scp from remote $_[0] to $target
my $target=$_[0];
my $return = '';
if ($type eq 'pdf') {
#system("pstotext $target $tempfile");
exec("/usr/local/bin/pdftotext $target $tempfile");
open(INFILE, "$tempfile");
@text=<INFILE>;
close(INFILE);
$return=join("",@text);
# $return='';
}
elsif ($type eq 'html') {
$return=`html2text $target`;
}
elsif ($type eq 'xls') {
$return=`xls2csv $target`;
}
elsif ($type eq 'doc') {
$return=`catdoc $target`;
}
elsif ($type eq 'txt') {
open(INFILE, "$target`");
@text=<INFILE>;
close(INFILE);
$return=join("",@text);
}
else {
$return = '';
}
#$content=~s/[^!-~]/ /g;
$return=~s/[^$list]/ /g; #replace all control characters with a space.
$return=~s/\'/\'\'/g; #escape all single quotes since they'll mess up the SQL.
$return=~s/\s+/ /g; #replace all instance of multiple spaces with single spaces.
@contents=();
$flag=0;
$m=0;
$nextstring=$return;
$lastspace=0;
while (!$flag) {
$lastspace = rindex($nextstring,' ',2000); #this returns the last instance of a space before 2000 chars
$contents[$m]=substr($nextstring,0,$lastspace); #grabs the string until the last instance of a space before 2000 chars
$nextstring=substr($nextstring,$lastspace+1); #grabs the string from the last space to the end.
if (length($nextstring)<2000) {
$contents[$m+1]=$nextstring;
$flag=1;
}
$m++;
}
}