#!/usr/bin/perl
$Version = 'MythTV Recording Archiver v0.2';
# MythTV Recording Archiver
#
# Written by MrGandalf 03/01/07
sub usage
{
die <<"EndUsage";
$Version
Usage:
Required: (at least one of)
-archive : Run in archive mode.
-cleanup <mode> : Run in cleanup mode where <mode> is a comma seperated
list of abandoned, zerobyte, metadata, deleted or all
Archive Arguments:
-from <group> : Archive recordings from storage group <group>.
-to <group> : Archive recordings to storage group <group>.
Archive Options:
-ignorelivetv : Ignore LiveTV recordings.
-hostname <host> : Only archive recordings which exist on
Myth backend <host>.
Debug:
-debug : Don't archive or delete anything.
Cleanup Modes:
abandoned - Delete recordings with missing metadata.
zerobyte - Delete all zero-byte recordings and associated metadata.
metadata - Delete metadata for recordings which don't exist.
deleted - Delete recordins and metadata which exist in the 'Deleted'
storage group.
all - Perform all cleanup types.
Notes:
If -hostname option is not used, you must ensure that storage group
paths are identical across all MythTV backends.
EndUsage
}
use Getopt::Long;
use Carp qw(cluck);
use IO::File;
use DirHandle;
use DBI;
use File::Copy;
use strict;
# Pull in the MythTV module
use lib '/opt/mns/mythtv/share/perl';
use MythTV;
my $TRUE = 1;
my $FALSE = 0;
my $LIVETV_AUTOEXPIRE = 9999;
my $DELETED_RECGROUP = 'Deleted';
# Operational modes
my $MODE_ABANDONED = 0x01;
my $MODE_ZEROBYTE = 0x02;
my $MODE_METADATA = 0x04;
my $MODE_DELETED = 0x08;
my $MODE_ALL = 0x0f;
my $MODE_ARCHIVE = 0x10;
my $DBH;
my $MYTH;
my $_DEBUG_;
&main();
sub getArgs {
my $modes;
my $from;
my $to;
my $hostname;
my $ignorelivetv;
my $cleanupmode;
my $archivemode;
my $batchmode;
my $force;
my $debug;
my $p = new Getopt::Long::Parser;
if (!$p->getoptions('from=s' => \$from,
'to=s' => \$to,
'hostname=s' => \$hostname,
'ignorelivetv' => \$ignorelivetv,
'archive' => \$archivemode,
'cleanup=s' => \$cleanupmode,
'batch' => \$batchmode,
'force' => \$force,
'debug' => \$debug)) {
print "Invalid parameters given.\n";
usage();
}
$force = $TRUE if (defined($force));
$ignorelivetv = $TRUE if (defined($ignorelivetv));
$batchmode = $TRUE if (defined($batchmode));
$cleanupmode =~ tr/A-Z/a-z/;
$_DEBUG_ = $TRUE if (defined($debug));
if ($archivemode && (!$from || !$to)) {
print "Please specify -from and -to with -archive.\n";
usage();
}
if (defined($cleanupmode) && !$cleanupmode) {
print "Please specify one of abandoned, zerobyte, metadata, deleted or all with -cleanup.\n";
usage();
}
if ($cleanupmode) {
foreach my $mode (split(/\,/, $cleanupmode)) {
if ($mode eq 'abandoned') {
$modes |= $MODE_ABANDONED;
} elsif ($mode eq 'zerobyte') {
$modes |= $MODE_ZEROBYTE;
} elsif ($mode eq 'metadata') {
$modes |= $MODE_METADATA;
} elsif ($mode eq 'deleted') {
$modes |= $MODE_DELETED;
} elsif ($mode eq 'all') {
$modes |= $MODE_ALL;
} else {
print "Invalid mode '$mode' specified.\n";
usage();
}
}
}
if (defined($archivemode)) {
$modes |= $MODE_ARCHIVE;
}
if (($from eq 'LiveTV') && !$ignorelivetv && !$force) {
print "Your source group is $from but you are not ignoring LiveTV recordings, ".
"please use -force option.\n";
exit 1;
}
return($modes, $from, $to, $hostname, $ignorelivetv, $batchmode);
}
sub main {
my($modes, $from, $to, $hostname, $ignorelivetv, $batchmode) = getArgs();
my $samefs = $FALSE;
my %recordingLookup;
my %abandoned;
print modesToString($modes);
$MYTH = new MythTV();
$DBH = $MYTH->{'dbh'};
my $storage = getStorageGroups($hostname);
my $recordings = getRecordings($hostname);
my($stored, $cifs, $thumbs) = getStoredRecordings($storage);
if ($modes & $MODE_ARCHIVE) {
my $fromDir = $$storage{$from};
my $toDir = $$storage{$to};
die("FATAL: Storage group '$from' is not defined in database, aborting!\n") if (!$fromDir);
die("FATAL: Storage group '$to' is not defined in database, aborting!\n") if (!$toDir);
my($mpFrom, undef, undef, undef) = getMountpoint($fromDir);
my($mpTo, undef, undef, undef) = getMountpoint($toDir);
if ($mpFrom eq $mpTo) {
print "From and to storage groups reside on the same filesystem, ".
"moving recordings rather than copying/deleting.\n";
$samefs = $TRUE;
}
}
# Some recordings may have moved, figure out which ones and update them
foreach my $group (keys %{$recordings}) {
foreach my $recording (keys %{$$recordings{$group}}) {
my $chanid = $$recordings{$group}->{$recording}->{'CHANID'};
my $starttime = $$recordings{$group}->{$recording}->{'START'};
my $title = $$recordings{$group}->{$recording}->{'TITLE'};
my $subtitle = $$recordings{$group}->{$recording}->{'SUBTITLE'};
$recordingLookup{$recording}++;
my $_group = findRecording($storage, $recording, $chanid, $starttime);
if (!defined($_group)) {
if ($modes & $MODE_METADATA) {
print "\tDeleting meta-data for missing recording '$recording' in group '$group'.\n";
deleteFromDb($chanid, $starttime, $recording, $group);
} else {
print "\tStorage group '$group': problem found with recording title '$title - ".
"$subtitle'.\n";
}
next;
}
updateStorageGroup($chanid, $starttime, $recording, $group)
if (defined($_group) && ($group ne $_group));
}
}
# Update
$recordings = getRecordings($hostname);
# Catch recording files still lingering around
my %totalSize;
foreach my $group (keys %{$stored}) {
foreach my $recording (keys %{$$stored{$group}}) {
if (!$recordingLookup{$recording}) {
my $size = $$stored{$group}->{$recording};
$abandoned{$group}->{$recording} = $size;
print "WARNING: Found recording file '$recording' in storage group '$group'\n ".
"of size $size with no entry in database!\n";
$totalSize{$group} += $size;
}
}
}
foreach my $group (keys %{$cifs}) {
foreach my $recording (keys %{$$cifs{$group}}) {
my $size = $$cifs{$group}->{$recording};
$abandoned{$group}->{$recording} = $size;
print "WARNING: Found recording file '$recording' in storage group '$group'\n ".
"of size $size with no entry in database!\n";
$totalSize{$group} += $size;
}
}
foreach my $group (keys %totalSize) {
my $size = $totalSize{$group};
print "Found a total of $size bytes in storage group '$group' with no database entries!\n";
}
if ($modes & $MODE_ABANDONED) {
if (!$batchmode) {
print "Preparing to cleanup recordings, press ctrl-c within 10 seconds to abort.\n";
sleep 10;
}
foreach my $group (keys %abandoned) {
foreach my $recording (keys %{$abandoned{$group}}) {
my $size = $abandoned{$group}->{$recording};
print "\tDeleting recording '$recording' in storage group '$group' of size $size bytes..\n";
unlinkFile($$storage{$group}, $recording);
if ($$thumbs{$recording}) {
foreach my $thumb (@{$$thumbs{$recording}}) {
print "\t\tDeleting thumbnail '$thumb'..\n";
unlinkFile($$storage{$group}, $thumb);
}
}
}
}
if ($modes & $MODE_DELETED) {
# Take care of recordings in the 'Deleted' group.
foreach my $group (keys %{$recordings}) {
foreach my $recording (keys %{$$recordings{$group}}) {
my $recgroup = $$recordings{$group}->{$recording}->{'RECGROUP'};
my $chanid = $$recordings{$group}->{$recording}->{'CHANID'};
my $starttime = $$recordings{$group}->{$recording}->{'START'};
if ($recgroup eq $DELETED_RECGROUP) {
print "\tDeleting recording '$recording' in storage group '$group' ".
"in recgroup '$recgroup'..\n";
unlinkFile($$storage{$group}, $recording);
deleteFromDb($chanid, $starttime, $recording, $DELETED_RECGROUP);
if ($$thumbs{$recording}) {
foreach my $thumb (@{$$thumbs{$recording}}) {
print "\t\tDeleting thumbnail '$thumb'..\n";
unlinkFile($$storage{$group}, $thumb);
}
}
}
}
}
}
}
foreach my $recording (keys %{$$recordings{$from}}) {
my $size = recordingSize($$storage{$from}, $recording);
my $chanid = $$recordings{$from}->{$recording}->{'CHANID'};
my $starttime = $$recordings{$from}->{$recording}->{'START'};
my $title = $$recordings{$from}->{$recording}->{'TITLE'};
my $subtitle = $$recordings{$from}->{$recording}->{'SUBTITLE'};
my $islivetv = $$recordings{$from}->{$recording}->{'LIVETV'};
$islivetv = 'LiveTV' if ($islivetv);
if ($size == 0) {
if ($modes & $MODE_ZEROBYTE) {
my $recgroup = $$recordings{$from}->{$recording}->{'RECGROUP'};
print "\tDeleting zero-byte recording '$recording' in storage group '$from' ".
"in recgroup '$recgroup'..\n";
unlinkFile($$storage{$from}, $recording);
deleteFromDb($chanid, $starttime, $recording, $from);
if ($$thumbs{$recording}) {
foreach my $thumb (@{$$thumbs{$recording}}) {
print "\t\tDeleting thumbnail '$thumb'..\n";
unlinkFile($$storage{$from}, $thumb);
}
}
} else {
print "WARNING: File '$recording' for $islivetv recording of '$title - $subtitle' ".
"is 0 bytes in size!\n";
}
} elsif ($modes & $MODE_ARCHIVE) {
if (!$ignorelivetv || ($ignorelivetv && !$islivetv)) {
my $fromDir = $$storage{$from};
my $toDir = $$storage{$to};
print "\tArchiving recording '$recording':\n\t\tTitle '$title - $subtitle' from ".
"'$fromDir' to '$toDir'..\n";
if (!archiveFile($fromDir, $toDir, $recording, $samefs)) {
updateStorageGroup($chanid, $starttime, $recording, $to);
}
if ($$thumbs{$recording}) {
foreach my $thumb (@{$$thumbs{$recording}}) {
archiveFile($fromDir, $toDir, $thumb, $samefs);
}
}
}
}
}
# Now print some stats..
print "\n\tStorage Group\tSize\tPortion\n";
print "\t----------------------------------------\n";
my $total = 0;
foreach my $group (keys %{$recordings}) {
my $group_s = 0;
my(undef, $mp_size, undef, undef) = getMountpoint($$storage{$group});
$mp_size = $mp_size * 1024;
$mp_size -= (.05 * $mp_size);
foreach my $recording (keys %{$$recordings{$group}}) {
my $size = recordingSize($$storage{$group}, $recording);
$group_s += $size;
}
my $portion = round((($group_s / $mp_size) * 100), 2) if ($mp_size);
print "\t$group\t\t$group_s\t$portion%\n";
$total += $group_s;
}
print "\t----------------------------------------\n";
print "\tTotal:\t\t$total\n\n";
print "All done!\n";
}
sub getStorageGroups {
my $hostname = shift;
my %groups;
my $where;
$where = "WHERE hostname = '$hostname'" if ($hostname);
my $sth = $DBH->prepare("SELECT groupname, dirname FROM storagegroup $where");
$sth->execute();
while (my $row = $sth->fetchrow_hashref()) {
my $path = $row->{'dirname'};
$path =~ s/\/+$//;
$groups{$row->{'groupname'}} = $path;
}
$sth->finish();
return \%groups;
}
sub getRecordings {
my $hostname = shift;
my %recordings;
my $where;
$where = "WHERE hostname = '$hostname'" if ($hostname);
my $sth = $DBH->prepare("SELECT chanid, starttime, basename, title, subtitle, ".
"autoexpire, storagegroup, recgroup FROM recorded $where");
$sth->execute();
while (my $row = $sth->fetchrow_hashref()) {
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'TITLE'} =
$row->{'title'};
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'SUBTITLE'} =
$row->{'subtitle'};
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'CHANID'} =
$row->{'chanid'};
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'START'} =
$row->{'starttime'};
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'RECGROUP'} =
$row->{'recgroup'};
$recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'LIVETV'} =
($row->{'autoexpire'} >= $LIVETV_AUTOEXPIRE) ? $TRUE : $FALSE;
}
$sth->finish();
return \%recordings;
}
sub getStoredRecordings {
my $storage = shift;
my %stored;
my %cifs;
my %thumbs;
foreach my $group (keys %{$storage}) {
my $path = $$storage{$group};
my $dir = new DirHandle($path);
die("FATAL: Directory '$path' for storage group '$group' does not exist!\n")
if (!defined($dir));
while (defined(my $entry = $dir->read())) {
next if (-d $entry);
next if (($entry !~ /\.mpg$/i) &&
($entry !~ /\.nuv$/i) &&
($entry !~ /\.png$/i) &&
($entry !~ /^cifs/));
if ($entry =~ /\.mpg$/) {
$stored{$group}->{$entry} = recordingSize($path, $entry);
} elsif ($entry =~ /\.png$/) {
if ($entry =~ /^(.+\.(mpg|nuv))\./i) {
push @{$thumbs{$1}}, $entry;
}
} elsif ($entry =~ /^cifs/) {
$cifs{$group}->{$entry} = recordingSize($path, $entry);
}
}
}
return(\%stored, \%cifs, \%thumbs);
}
sub updateStorageGroup {
my $chanid = shift;
my $starttime = shift;
my $recording = shift;
my $group = shift;
my $sql = "UPDATE recorded SET storagegroup = '$group' ".
"WHERE chanid = $chanid AND starttime = '$starttime' AND basename = '$recording'";
if ($_DEBUG_) {
print "DBG: updateStorageGroup(): $sql\n";
} else {
my $sth = $DBH->prepare($sql);
$sth->execute();
$sth->finish();
}
}
sub findRecording {
my $storage = shift;
my $recording = shift;
my $chanid = shift;
my $starttime = shift;
my $found;
foreach my $group (keys %{$storage}) {
my $directory = $$storage{$group};
if (-e "$directory/$recording") {
if ($found) {
print "WARNING: Found recording '$recording' in multiple groups, ignoring!\n";
return undef;
}
$found = $group;
}
}
if (!$found) {
print "WARNING: Unable to find recording '$recording' in any storage group!\n";
return undef;
}
return $found;
}
sub deleteFromDb {
my $chanid = shift;
my $starttime = shift;
my $basename = shift;
my $recgroup = shift;
doSql("DELETE FROM recorded WHERE chanid = $chanid AND starttime = '$starttime' AND ".
"basename = '$basename' AND recgroup = '$recgroup'");
doSql("DELETE FROM recordedcredits WHERE chanid = $chanid AND starttime = '$starttime'");
doSql("DELETE FROM recordedmarkup WHERE chanid = $chanid AND starttime = '$starttime'");
doSql("DELETE FROM recordedprogram WHERE chanid = $chanid AND starttime = '$starttime'");
doSql("DELETE FROM recordedrating WHERE chanid = $chanid AND starttime = '$starttime'");
doSql("DELETE FROM recordedseek WHERE chanid = $chanid AND starttime = '$starttime'");
}
sub archiveFile {
my $fromDir = shift;
my $toDir = shift;
my $file = shift;
my $samefs = shift;
if (!-e "$fromDir/$file") {
print "WARNING: Attempting to archive the file '$fromDir/$file' which doesn't exist!\n";
return $TRUE;
}
if (-e "$toDir/$file") {
print "WARNING: Recording '$file' already exists in destination group, ignoring!\n";
return $TRUE;
}
if (!$samefs && !checkForSpace($fromDir, $toDir, $file)) {
print "Storage group directory '$toDir' is too full to achive recording '$file', ignoring.\n";
return $TRUE;
}
if ($_DEBUG_) {
print "DBG: Would archive '$fromDir/$file' to '$toDir/$file' mode ".($samefs ? "mv" : "cp")."\n";
} else {
if ($samefs) {
if (!move("$fromDir/$file", "$toDir/$file")) {
print "WARNING: Failed to archive/move recording $file from $fromDir to $toDir: $!\n";
return $TRUE;
}
} else {
if (!copy("$fromDir/$file", "$toDir/$file")) {
print "WARNING: Failed to archive/copy recording $file from $fromDir to $toDir: $!\n";
return $TRUE;
}
}
}
my $orig_size = recordingSize($fromDir, $file);
my $copy_size = recordingSize($toDir, $file);
if (!$samefs && ($orig_size != $copy_size)) {
print "WARNING: Failed to copy recording '$file' from $fromDir to $toDir, file sizes don't match!\n";
return $TRUE;
} elsif (!$samefs) {
unlinkFile($fromDir, $file);
}
return $FALSE;
}
sub unlinkFile {
my $fromDir = shift;
my $file = shift;
return $TRUE if (!$fromDir || !$file);
if (-e "$fromDir/$file") {
if ($_DEBUG_) {
print "DBG: Would unlink \"$fromDir/$file\"\n";
} else {
if (!unlink "$fromDir/$file") {
print "WARNING: Failed to unlink file $file in $fromDir: $!\n";
return $TRUE;
}
}
}
return $FALSE;
}
sub isFileInUse {
my $file = shift;
my $result = `fuser $file`;
$result =~ s/\s+$//;
my(undef, $pids) = split(/\:/, $result);
return $pids;
}
sub checkForSpace {
my $fromDir = shift;
my $toDir = shift;
my $file = shift;
my $mount;
my $available;
my $percentage;
my(undef, undef, $available, $percentage) = getMountpoint($toDir);
my $file_size = recordingSize($fromDir, $file);
return $FALSE if (($available < $file_size) || ($percentage > 98));
return $TRUE;
}
sub getMountpoint {
my $directory = shift;
my $mp;
my $size;
my $free;
my $percentage;
my $buff = `df -kP $directory`;
foreach my $line (split(/\n/, $buff)) {
($mp, $size, undef, $free, $percentage) = split(/\s+/, $line, 5)
if ($line !~ /^Filesystem/);
}
$free = $free * 1024;
$percentage =~ s/\%//;
return($mp, $size, $free, $percentage);
}
sub recordingSize {
my $directory = shift;
my $recording = shift;
if (!-e "$directory/$recording") {
return -1;
}
my $size = -s "$directory/$recording";
return $size;
}
sub round {
my $num = shift;
my $place = shift;
my($whole, $decimal) = split(/\./, $num);
my $decimal_t = substr($decimal, 0, $place);
if ($decimal_t) {
my $rounder = substr($decimal, $place, 1);
$decimal_t += 1 if ($rounder && $rounder > 5);
} else {
$decimal_t = 0;
}
return $whole.".".$decimal_t;
}
sub modesToString {
my $modes = shift;
my $string = "\nWorking in the following modes:\n";
if ($modes & $MODE_ARCHIVE) {
$string .= "\t-> Archive Recordings\n";
}
if ($modes & $MODE_ABANDONED) {
$string .= "\t-> Delete abandoned recordings\n";
}
if ($modes & $MODE_ZEROBYTE) {
$string .= "\t-> Delete zero-byte recordings\n";
}
if ($modes & $MODE_METADATA) {
$string .= "\t-> Delete meta-data for missing recordings\n";
}
if ($modes & $MODE_DELETED) {
$string .= "\t-> Delete recordings in 'Deleted' group\n";
}
$string .= "\n";
return $string;
}
sub doSql {
my $sql = shift;
if ($_DEBUG_) {
print "DBG: $sql\n";
} else {
my $sth = $DBH->prepare($sql);
$sth->execute();
$sth->finish();
}
}