. cg_filecomp
cg_geturl () {
GETBASEURL=$1
GETOBJECT=$2
DESTDIR=$3
DESTFILE=$4
QUALITY=$5
ARCHIVE_FLAG=$6
TIMESTAMP_FLAG=$7
if [ -z $CG_GETURL_CACHE ]
then
CG_GETURL_CACHE="/capture/tmp/`whoami`.cg_geturl.cache"
fi # if [ -z $CG_GETURL_CACHE ]
if [ ! -d $CG_GETURL_CACHE ]
then
echo "No cache for \$CG_GETURL_CACHE found creating $CG_GETURL_CACHE"
echo "You might want to create one somewhere pointed to by the environment variable CG_GETURL_CACHE"
echo "Example export CG_GETURL_CACHE=/var/local/cache/cg_geturl"
mkdir -v $CG_GETURL_CACHE
mkdir -v $CG_GETURL_CACHE/last
fi # [ ! -d $CG_GETURL_CACHE ]
#
#---- Use squid if available
#
squid_running="`ps -C squid|grep squid|wc -l`"
if [ $squid_running == "0" ]
then
unset http_proxy
unset ftp_proxy
else
export http_proxy=http://127.0.0.1:3128
export ftp_proxy=$http_proxy
fi
cd $CG_GETURL_CACHE >/dev/null 2>&1 # Need to define this otherwise it will dump all to /tmp Use something like /var/local/cache/cg_geturl with a+rwx permits
if [ ! -f ./last/$DESTFILE.Content_Length.last ]
then
echo "Getting $GETBASEURL$GETOBJECT for the first time"
echo "Content_Length: \"0\"" > ./last/$DESTFILE.Content_Length.last #create a dummy last Content_Length file
fi
orgContent_Length=" "
newContent_Length=" "
orgContent_Length="`cat ./last/$DESTFILE.Content_Length.last | awk '{print $2}'`"
starttime=`getnanotime`
proc_getdatetime=`getdatetime`
curl -k --connect-timeout 30 --retry-delay 11 --max-time 60 --retry-max-time 60 --retry 2 --silent --show-error --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/null
newContent_Length="`cat $DESTFILE.Content_Length | awk '{print $2}'`"
if [ $orgContent_Length == $newContent_Length ]
then
echo "Unchanged $DESTFILE `date`"
if [ -f $DESTDIR/$DESTFILE ]
then
cg_filecomp $DESTFILE $DESTDIR/$DESTFILE
if [ $RC != "1" ]
then
rm -f $DESTDIR/$DESTFILE
cp -p $DESTFILE $DESTDIR/$DESTFILE
fi
else
cp -p $DESTFILE $DESTDIR/$DESTFILE
fi # if [ -f $DESTDIR/$DESTFILE ]
else
if [ -f $DESTFILE.Content_Length ]
then
mv -f $DESTFILE.Content_Length ./last/$DESTFILE.Content_Length.last
fi
if [ -f $DESTFILE ]
then
mv -f $DESTFILE $DESTFILE.save
fi
# This might be timing out on large images Modified --max-time from 10 to 20 6/24/2006
curl -k --connect-timeout 30 --retry-delay 10 --max-time 240 --retry-max-time 120 --retry 2 --silent --show-error $GETBASEURL$GETOBJECT > $DESTFILE # Use -S flag and rewget if an error
cg_filecomp $DESTFILE $DESTFILE.save
if [ ! $RC == "1" ]
then # New file
echo "Updating $GETBASEURL$DESTFILE `date`..."
mv -f $DESTFILE.save ./last/$DESTFILE
rm -f $DESTDIR/last/$DESTFILE
cp -pf ./last/$DESTFILE $DESTDIR/last/$DESTFILE
rm -f $DESTDIR/$DESTFILE
cp -pf $DESTFILE $DESTDIR/$DESTFILE
if [ $ARCHIVE_FLAG == "Y" ]
then
BASEFILE="`basename $DESTFILE |\
sed 's/\.jpg//'|\
sed 's/\.JPG//'|\
sed 's/\.GIF//'|\
sed 's/\.gif//'|\
sed 's/\.png//'`"
if [ ! -d $DESTDIR/archive ]
then
mkdir -v $DESTDIR/archive
fi # if [ ! -d $DESTDIR/archive ]
if [ ! -d $DESTDIR/archive/$BASEFILE ]
then
mkdir -v $DESTDIR/archive/$BASEFILE
fi # if [ ! -d $DESTDIR/archive/$BASEFILE ]
nowdate=`date +%Y%m%d%H%M%S`
rm -f $BASEFILE.$nowdate.jpg
#---- Force image to jpeg Need to add test to see if it is already jpg maybe use file $DESTDIR/$DESTFILE
convert -format jpg -quality $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg
rm $DESTDIR/$DESTFILE
cp -p $BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
#
#---- Do a final check that the image isn't already archived based on the sha1sum. md5sum is probably good enough, but heck, we got cpu to burn.
#
sha1sum $BASEFILE.$nowdate.jpg > $BASEFILE.$nowdate.sha1sum
sha1val=`cat $BASEFILE.$nowdate.sha1sum|awk '{print $1}'`
destbase=`basename $DESTDIR/archive/$BASEFILE`
if [ -f $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt ]
then
nfound=`grep -c --mmap $sha1val $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt`
if [ $nfound -eq 0 ]
then
cat $BASEFILE.$nowdate.sha1sum >> $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt
#---- Add timestamp to upper left if requested
if [ $TIMESTAMP_FLAG == "Y" ]
then
convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
rm -f $BASEFILE.$nowdate.jpg
cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
else
mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
fi
echo "Archived $BASEFILE.$nowdate.jpg"
#
#---- Time statistics
#
endtime=`getnanotime`
proctime=`echo 8 $endtime $starttime|bc /usr/local/bc/subfloat.bc`
echo "STATS 1"
if [ -f $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg ]
then
filesize=`ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg|awk '{print $5}'`
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
else
echo "Can't archive $BASEFILE.$nowdate.jpg - NOT FOUND"
ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
fi
else
echo "$BASEFILE.$nowdate.jpg was already archived"
fi # if [ $nfound -ne 0 ]
else
cat $BASEFILE.$nowdate.sha1sum >> $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt
if [ $TIMESTAMP_FLAG == "Y" ]
then
convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
rm -f $BASEFILE.$nowdate.jpg
cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
else
mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
fi
mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
echo "Archived $BASEFILE.$nowdate.jpg"
#
#---- Time statistics
#
endtime=`getnanotime`
proctime=`echo 8 $endtime $starttime|bc /usr/local/bc/subfloat.bc`
echo "STATS 2"
if [ -f $DESTDIR/archive/$BASEFILE ]
then
filesize=`ls -l $DESTDIR/archive/$BASEFILE|awk '{print $5}'`
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
else
echo "Can't archive $BASEFILE.jpg - NOT FOUND"
ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE
fi
fi # if [ -f $DESTDIR/archive/$BASEFILE/destbase.sha1list.txt ]
else # Not archived Clean up but need better fix
rm -f $BASEFILE.$nowdate.jpg
fi # if [ $ARCHIVE_FLAG == "Y" ]
else # File did not really change
echo "Unchanged (False change) $DESTFILE `date`"
fi # if [ ! $RC == "1" ]
fi # if [ $orgContent_Length == $newContent_Length ] else
rm -f $BASEFILE.$nowdate.sha1sum
cd - >/dev/null 2>&1
}