rendered paste body##---- Function: cg_geturl - Download an image if it hasn't changed (According to Content-Length)## Copyright: John B. Battles 2006# License: GPL 2#mk# Currently using md5sum# TODO LIST:# 1. Clean up cache create# 2. Set max cache maybe have something compress the cache periodically then uncompress if it is compressed# 3. Use a Cache hierarchy maybe by website this should speed up access in a reiserfs# 4. Process 404 and other failed errors# 5. ETag is not good if site is mirrored Use length may need to use Last Modified must be newer# 6. Create a cg_geturl.conf for font etcchecksum=md5. cg_filecompcg_geturl () {GETBASEURL=$1GETOBJECT=$2DESTDIR=$3DESTFILE=$4QUALITY=$5ARCHIVE_FLAG=$6TIMESTAMP_FLAG=$7#echo " GETBASEURL <$GETBASEURL> GETOBJECT <$GETOBJECT> DESTDIR <$DESTDIR> DESTFILE <$DESTFILE> QUALITY <$QUALITY> ARCHIVE_FLAG <$ARCHIVE_FLAG>"if [ -z $CG_GETURL_CACHE ]then CG_GETURL_CACHE="/capture/tmp/`whoami`.cg_geturl.cache"fi # if [ -z $CG_GETURL_CACHE ]if [ ! -d $CG_GETURL_CACHE ]then echo "No cache for \$CG_GETURL_CACHE found creating $CG_GETURL_CACHE" echo "You might want to create one somewhere pointed to by the environment variable CG_GETURL_CACHE" echo "Example export CG_GETURL_CACHE=/var/local/cache/cg_geturl" mkdir -v $CG_GETURL_CACHE mkdir -v $CG_GETURL_CACHE/lastfi # [ ! -d $CG_GETURL_CACHE ]squid_running="`ps -C squid|grep squid|wc -l`"if [ $squid_running == "0" ]then unset http_proxy unset ftp_proxyelse export http_proxy=http://127.0.0.1:3128 export ftp_proxy=$http_proxy ficd $CG_GETURL_CACHE >/dev/null 2>&1 # Need to define this otherwise it will dump all to /tmp Use something like /var/local/cache/cg_geturl with a+rwx permitsif [ ! -f ./last/$DESTFILE.Content_Length.last ]then echo "Getting $GETBASEURL$GETOBJECT for the first time" echo "Content_Length: \"0\"" > ./last/$DESTFILE.Content_Length.last #create a dummy last Content_Length filefiorgContent_Length=" "newContent_Length=" "orgContent_Length="`cat ./last/$DESTFILE.Content_Length.last | awk '{print $2}'`" starttime=`getnanotime` proc_getdatetime=`getdatetime`curl -k --connect-timeout 30 --retry-delay 11 --max-time 60 --retry-max-time 60 --retry 2 --silent --show-error --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/null#curl -k --connect-timeout 10 --retry-delay 5 --max-time 10 --retry-max-time 5 --retry 2 --show-error --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/nullnewContent_Length="`cat $DESTFILE.Content_Length | awk '{print $2}'`"#echo "orgContent_Length <$orgContent_Length>"#echo "newContent_Length <$newContent_Length>"if [ $orgContent_Length == $newContent_Length ]then echo "Unchanged $DESTFILE `date`" if [ -f $DESTDIR/$DESTFILE ] then cg_filecomp $DESTFILE $DESTDIR/$DESTFILE if [ $RC != "1" ] then rm -f $DESTDIR/$DESTFILE cp -p $DESTFILE $DESTDIR/$DESTFILE fi else cp -p $DESTFILE $DESTDIR/$DESTFILE fi # if [ -f $DESTDIR/$DESTFILE ]else if [ -f $DESTFILE.Content_Length ] then mv -f $DESTFILE.Content_Length ./last/$DESTFILE.Content_Length.last fi if [ -f $DESTFILE ] then mv -f $DESTFILE $DESTFILE.save fi# rm -f $GETOBJECT# This might be timing out on large images Modified --max-time from 10 to 20 6/24/2006 curl -k --connect-timeout 30 --retry-delay 10 --max-time 240 --retry-max-time 120 --retry 2 --silent --show-error $GETBASEURL$GETOBJECT > $DESTFILE # Use -S flag and rewget if an error# if [ $GETOBJECT != $DESTFILE ]# then# mv -f $GETOBJECT $DESTFILE# fi cg_filecomp $DESTFILE $DESTFILE.save if [ ! $RC == "1" ] then # New file echo "Updating $GETBASEURL$DESTFILE `date`..." mv -f $DESTFILE.save ./last/$DESTFILE rm -f $DESTDIR/last/$DESTFILE cp -pf ./last/$DESTFILE $DESTDIR/last/$DESTFILE rm -f $DESTDIR/$DESTFILE cp -pf $DESTFILE $DESTDIR/$DESTFILE if [ $ARCHIVE_FLAG == "Y" ] then BASEFILE="`basename $DESTFILE |\ sed 's/\.jpg//'|\ sed 's/\.JPG//'|\ sed 's/\.GIF//'|\ sed 's/\.gif//'|\ sed 's/\.png//'`" if [ ! -d $DESTDIR/archive ] then mkdir -v $DESTDIR/archive fi # if [ ! -d $DESTDIR/archive ] if [ ! -d $DESTDIR/archive/$BASEFILE ] then mkdir -v $DESTDIR/archive/$BASEFILE fi # if [ ! -d $DESTDIR/archive/$BASEFILE ] #echo "ARCFILE <$ARCFILE>" nowdate=`date +%Y%m%d%H%M%S` rm -f $BASEFILE.$nowdate.jpg##---- Do a final check that the image isn't already archived based on the md5sum##echo "convert $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg" convert -format jpg -quality $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg rm $DESTDIR/$DESTFILE cp -p $BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE #Fix up if orig file was not jpg md5sum $BASEFILE.$nowdate.jpg > $BASEFILE.$nowdate.md5sum md5val=`cat $BASEFILE.$nowdate.md5sum|awk '{print $1}'` destbase=`basename $DESTDIR/archive/$BASEFILE` if [ -f $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt ] then nfound=`grep -c --mmap $md5val $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt` if [ $nfound -eq 0 ] then cat $BASEFILE.$nowdate.md5sum >> $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt # convert -draw "text 5,14 '`date`'" -font helvetica -pointsize 16 -fill yellow $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE# rm -f $BASEFILE.$nowdate.jpg if [ $TIMESTAMP_FLAG == "Y" ] then convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg rm -f $BASEFILE.$nowdate.jpg cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE else mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE fi echo "Archived $BASEFILE.$nowdate.jpg" ##---- Time statistics# endtime=`getnanotime` proctime=`echo 8 $endtime $starttime|bc /usr/local/bc/subfloat.bc` echo "STATS 1" if [ -f $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg ] then filesize=`ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg|awk '{print $5}'` echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" else echo "Can't archive $BASEFILE.$nowdate.jpg - NOT FOUND" ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg fi else echo "$BASEFILE.$nowdate.jpg was already archived" fi # if [ $nfound -ne 0 ] else cat $BASEFILE.$nowdate.md5sum >> $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt if [ $TIMESTAMP_FLAG == "Y" ] then convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg rm -f $BASEFILE.$nowdate.jpg cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE else mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE fi# convert -draw "text 5,14 '`date`'" -font helvetica -pointsize 16 -fill yellow $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE# rm -f $BASEFILE.$nowdate.jpg mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE echo "Archived $BASEFILE.$nowdate.jpg"##---- Time statistics# endtime=`getnanotime` proctime=`echo 8 $endtime $starttime|bc /usr/local/bc/subfloat.bc` echo "STATS 2" if [ -f $DESTDIR/archive/$BASEFILE ] then filesize=`ls -l $DESTDIR/archive/$BASEFILE|awk '{print $5}'` echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" else echo "Can't archive $BASEFILE.jpg - NOT FOUND" ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE fi fi # if [ -f $DESTDIR/archive/$BASEFILE/destbase.md5list.txt ] else # Not archived Clean up but need better fix rm -f $BASEFILE.$nowdate.jpg fi # if [ $ARCHIVE_FLAG == "Y" ] else # File did not really chage echo "Unchanged (False change) $DESTFILE `date`" fi # if [ ! $RC == "1" ]fi # if [ $orgContent_Length == $newContent_Length ] elserm -f $BASEFILE.$nowdate.md5sumcd - >/dev/null 2>&1}