rendered paste body# Patch to speed up apt-file searches. My test searches went from ~14 seconds to 4:
#
#### Normal string matches:
#
# % date ; /usr/bin/apt-file search nvidia | wc ; date
# Fri Feb 17 01:56:51 EST 2006
# 206 412 15018
# Fri Feb 17 01:57:06 EST 2006
#
# % date ; perl ./apt-file -p search nvidia | wc ; date
# Fri Feb 17 01:57:16 EST 2006
# 206 412 15018
# Fri Feb 17 01:57:20 EST 2006
#
#### Regex matches:
#
# % date ; /usr/bin/apt-file -F -x search "//ldap\.h" | wc ; date
# Fri Feb 17 01:59:30 EST 2006
# 3 6 147
# Fri Feb 17 01:59:44 EST 2006
#
# % date ; perl ./apt-file -F -p -x search "//ldap\.h" | wc ; date
# Fri Feb 17 02:00:15 EST 2006
# 3 6 147
# Fri Feb 17 02:00:19 EST 2006
--- /usr/bin/apt-file 2005-05-09 09:06:28.000000000 -0400
+++ apt-file 2006-02-17 01:18:36.000000000 -0500
@@ -174,31 +174,96 @@
}
}
+# Use zpcregrep instead of looping through zcat output.
+# This method is faster than the zcat one but still slower than POSIX.
+sub pcre_gzip_grep($$$) {
+ my ($file, $regexp, $is_file_pattern) = @_;
+
+ # Use pzgrep since it is faster than zcat and parsing with perl.
+ my $pzgrep_cmd = "zpcregrep ";
+ if ($Conf->{ignore_case}) {
+ $pzgrep_cmd .= "-i ";
+ }
+ $pzgrep_cmd .= "-- ";
+ if ($is_file_pattern) {
+ # Package names cannot have spaces (not sure if in Debian policy).
+ $pzgrep_cmd .= "'${regexp}\\s+\\S+\\s*\$'";
+ } else {
+ # Because packages could be "foo,ourpackage,bar", we need to add
+ # extra regex. Also, filenames can have spaces.
+ $pzgrep_cmd .= "'^\\S+.*\\s+\\S+?${regexp}\\S+?\\s*\$'";
+ }
+ $pzgrep_cmd .= " $file |";
+
+ return $pzgrep_cmd;
+}
+
+# This is much faster than using perl style regex.
+sub posix_gzip_grep($$$) {
+ my ($file, $regexp, $is_file_pattern) = @_;
+
+ # Use zegrep since it is faster than zcat and parsing with perl.
+ my $zegrep_cmd = "zegrep ";
+ if ($Conf->{ignore_case}) {
+ $zegrep_cmd .= "-i ";
+ }
+ $zegrep_cmd .= "-- ";
+ if ($is_file_pattern) {
+ # Package names cannot have spaces (not sure if in Debian policy).
+ $zegrep_cmd .= "'${regexp}[[:space:]]+[^[:space:]]+[[:space:]]*\$'";
+ } else {
+ # Because packages could be "foo,ourpackage,bar", we need to add
+ # extra regex. Also, filenames can have spaces.
+ $zegrep_cmd .= "'^[^[:space:]]+.*[[:space:]]+[^[:space:]]+?${regexp}[^[:space:]]+?[[:space:]]*\$'";
+ }
+ $zegrep_cmd .= " $file |";
+
+ return $zegrep_cmd;
+}
+
+
sub do_grep($$) {
- my ($data, $pattern) = @_;
+ my ($data, $is_file_pattern) = @_;
my $ret;
- my ($pack, $file);
- debug "regexp: $pattern";
+ my ($packages, $file);
$|=1;
- my $regexp = eval { $Conf->{ignore_case} ? qr/$pattern/i : qr/$pattern/ };
+
+ my $regexp = join "", (
+ $Conf->{pattern},
+ $is_file_pattern && $Conf->{fixed_strings} ? "" : ".*?",
+ );
+ debug qq[Regexp = "$regexp"\n];
error($@) if $@;
+
foreach(@$data) {
my $file = "$Conf->{cache}/$_->{dest}";
next if (! -f $file);
$file = quotemeta $file;
debug "Search in $file";
- open (ZCAT, "zcat $file |") ||
- warning "Can't zcat $file";
- while(<ZCAT>) {
- next if ! (($pack, $file) = /$regexp/);
- debug_line ".";
- foreach (split /,/, $file) {
- push @{$ret->{$pack}}, basename $_;
+
+ my $zgrep_cmd = undef;
+
+ # Use perl by default for regex matches.
+ # Use POSIX by default for string matches since it is faster.
+ if (!$Conf->{is_regexp} || $Conf->{use_posix_grep}) {
+ $zgrep_cmd = posix_gzip_grep($file, $regexp, $is_file_pattern);
+ } else {
+ $zgrep_cmd = pcre_gzip_grep($file, $regexp, $is_file_pattern);
+ }
+
+ open (ZGREP, $zgrep_cmd)
+ || die qq[Cannot zgrep "$file" using "$zgrep_cmd": $!];
+ while(<ZGREP>) {
+ # Files may have spaces in the middle of them but a space at the
+ # end is not allowed.
+ warn "Error $_" if ! (($file, $packages) = /^(\S+(?:[\S+\s+]*\S+)?)\s+(\S+)\s*$/);
+ foreach (split /,/, $packages) {
+ push @{$ret->{$file}}, basename $_;
}
}
- close ZCAT;
- debug_line "\n";
+ close(ZGREP);
}
+
$ret = reverse_hash($ret);
if (!defined $Conf->{package_only}) {
foreach my $key (sort keys %$ret) {
@@ -213,29 +278,13 @@
}
sub grep_file($) {
- my $data = shift;
- my $pattern = join "", (
- '^(.*?',
- $Conf->{pattern},
- defined $Conf->{fixed_strings} ?
- ")" : '[^\s]*)',
- '\s+(\S+)\s*$',
- );
- do_grep $data, $pattern;
+ my $gzip_content_files = shift;
+ do_grep($gzip_content_files, 1);
}
sub grep_package($) {
- my $data = shift;
- my $pattern = join "", (
- '^(\S+)\s+',
- '(\S*/',
- $Conf->{pattern},
- defined $Conf->{fixed_strings} ?
- "" : ".*",
- ")",
- '$',
- );
- do_grep $data, $pattern;
+ my $gzip_content_files = shift;
+ do_grep($gzip_content_files, 0);
}
sub purge_cache($) {
@@ -273,9 +322,10 @@
--package-only -l Only display packages name
--fixed-string -F Do not expand pattern
--ignore-case -i Ignore case distinctions
- --regexp -x pattern is a regular expression
- --verbose -v run in verbose mode
- --dummy -y run in dummy mode (no action)
+ --regexp -x <pattern> pattern is a regular expression
+ --posix-grep -p Use POSIX egrep style regex (faster)
+ --verbose -v Run in verbose mode
+ --dummy -y Run in dummy mode (no action)
--help -h Show this help.
--version -V Show version number
@@ -298,6 +348,7 @@
"verbose|v" => \$Conf->{verbose},
"ignore-case|i" => \$Conf->{ignore_case},
"regexp|x" => \$Conf->{is_regexp},
+ "posix-grep|p" => \$Conf->{use_posix_grep},
"dummy|y" => \$Conf->{dummy},
"package-only|l" => \$Conf->{package_only},
"fixed-string|F" => \$Conf->{fixed_strings},
@@ -335,7 +386,7 @@
$Conf->{action} = shift @ARGV || "none";
$Conf->{pattern} = shift @ARGV;
if(defined $Conf->{pattern}) {
- $Conf->{pattern} =~ s/^\///;
+ $Conf->{pattern} =~ s,^/,,;
$Conf->{pattern} = quotemeta($Conf->{pattern}) unless $Conf->{is_regexp};
}
undef $!;
@@ -349,7 +400,7 @@
};
$Conf->{help}=2 if $Conf->{action} =~ m/search|list/ &&
- ! defined $Conf->{pattern};
+ ! defined $Conf->{pattern};
$Conf->{help}=2 if ! defined $actions->{$Conf->{action}} &&
! defined $Conf->{help};
print_help($Conf->{help}-1) if defined $Conf->{help};