#! /bin/bash########################################################################## Grabs newest movie trailers off http://www.apple.com/trailers and #### saves them on the disk for viewing. #### #### by Kevin Welch, 2005 ########################################################################### move to proper directoryPWD=`pwd`cd /home/kevin/sbin/trailersif [ -z "$1" ]; then # grab fresh copy of the trailer list for screenscraping echo "Grabbing fresh page..." wget -q http://www.apple.com/trailers/ -O trailers.html # scrape trailers.html for the "Newest Trailers" and "Exclusive Trailers" # columns and strip HTML: # resulting list is in format "DATE \t URL \t TITLE" echo "Filtering links..." sed -nr -e " /Trailers Exclusive/,/class=\"clearer\"/ { s/^\t*// s/<\/?div[^<]*>//g s/<\/?h[1-4]>//g s/<a href=\"/\thttp:\/\/www.apple.com\/trailers\// s/\">/\t/ s/<\/a>.*// p } " -e " /Newest Trailers/,/class=\"clearer\"/ { s/^\t*// s/<\/?div[^<]*>//g s/<\/?h[1-4]>//g s/<a href=\"/\thttp:\/\/www.apple.com\/trailers\// s/\">/\t/ s/<\/a>.*// p } " trailers.html | egrep ^[0-9]{2}[.][0-9]{2} | awk '{print $2}' > trailerlist.tmpelif [ -e $PWD/$1 ]; then # argument is filelist cp "$PWD/$1" trailerlist.tmpelif [[ "$1" == http* ]]; then # argument is url echo "$1" > trailerlist.tmpelse # Dunno what it is echo "Parameter not recognized." exit 1fi# loop through trailer listecho "Grabbing trailers..."numtrailers=`wc -l trailerlist.tmp | awk '{print $1}'`numhistory=0for url in `cat trailerlist.tmp`do # set temp short name for file name=`echo $url | sed -e 's/[.]html$//' | awk -F / '{print $6}'` # check if url is in history if [ -z "$1" ]; then if grep -q $url trailerhistory; then inhistory=TRUE else inhistory=FALSE fi else # force no history check for command-line parameters inhistory=FALSE fi if [[ $inhistory == TRUE ]]; then # it is; move on echo "History: $url" let "numhistory += 1" # w00! new trailer! grab it. else wget -q $url -O ./$name.html if grep -iq embed $name.html; then # we can look directly at the file # scour html file for <EMBED> tag, and nab reference quicktime mov refurl=`cat ./$name.html | sed -rne '/<(EMBED|embed|Embed)/,/<\/(EMBED|embed|Embed)>/p' | egrep -io '(HREF|SRC)=\"[^\"]*ref[^\"]*\"' | awk -F \" '{print $2}'` # we need to dig further into the directory to find an html file else wget -q $url/large.html -O large.html # check to see if large.html has an <EMBED> tag -- boy, does that make things easy if grep -iq embed large.html; then # yes, it does ... extract as before. echo -n "" # nope, large.html doesn't exist. We'll have to fish for it. else wget -q $url -O $name.html # look for images that are links: # search for "large" and "high" temp=`cat $name.html | sed -rne 's/^[ \t]*//;s/[ \t]*$//;/^$/d;s/parentalguide.org//;/<(a|A)/,/<\/(a|A)>/p' | egrep -i '<(a|A)[^<>]*(large|high|-lrg|_lrg|-lg|_lg|lg)' | egrep -io '<a[^<>]*' | egrep -io 'href="[^"]*(large|high|-lrg|_lrg|-lg|_lg|lg)[^"]*"' | awk -F \" '{print $2}'` if [[ `echo "$temp" | wc -l` -gt 1 ]]; then echo "More than one trailer present for $name... must be downloaded by hand." echo "$temp" largehtml="" else if [[ $url == *html ]]; then largehtml=`echo $url | sed -e "s/\/[^\/]*$/\/$temp/"` elif [[ $temp == http* ]]; then largehtml=$temp else largehtml=$url$temp fi fi [ -n "$largehtml" ] && wget -q "$largehtml" -O large.html > /dev/null fi # scour html file for <EMBED> tag, and nab reference quicktime mov refurl=`cat ./large.html | sed -rne '/<(EMBED|embed|Embed)/,/<\/(EMBED|embed|Embed)>/p' | egrep -io '(HREF|SRC)=\"[^\"]*mov\"' | grep -v 'apple.*images' | awk -F \" '{print $2}'` fi [ -n "$refurl" ] && wget -q $refurl -O $name-ref.mov # search reference quicktime mov for filenames [ -e $name-ref.mov ] && filename=`strings $name-ref.mov | grep '[.]mov' | sed -e 's/^[^a-z]//' | sed -ne '$p'` # create downloadable url and grab it, or if the file we downloaded is not a reference, move it if [ -n "$filename" ]; then downloadfile=`echo $refurl | sed "s/\/[^\/]*$/\/$filename/"` wget --spider $downloadfile 2> filesize.tmp filesizecommas=`cat filesize.tmp | grep Length | egrep -o '[0-9]+,[0-9]+,{0,1}[0-9]*'` filesize=`echo $filesizecommas | sed -e 's/,//g'` currentsize=0 echo -n "Downloading $filename [$filesizecommas bytes] ... 00%" wget -qb $downloadfile -O new/$filename > /dev/null sleep 2 while [ "$currentsize" -lt "$filesize" ] do currentsize=`du -b new/$filename | awk '{print $1}'` percent=$(($currentsize * 100 / $filesize )) [ $percent -lt 10 ] && percent=0$percent echo -ne "\b\b\b$percent%" sleep 1 done echo -ne "\b\b\b\bdone!\n" else [ -e $name-ref.mov ] && mv -v $name-ref.mov new/$name_m480.mov fi # add url to history file so we don't grab the trailer twice echo $url >> trailerhistory # clean up temp files [ -e $name.html ] && rm ./$name.html [ -e $name-ref.mov ] && rm ./$name-ref.mov [ -e large.html ] && rm ./large.html fidone# final outputnumdownloaded=$(( $numtrailers - $numhistory ))if [[ "$numdownloaded" == "0" ]]; then echo "No new trailers."else echo "Downloaded $numdownloaded new trailers."fi# clean up[ -e filesize.tmp ] && rm filesize.tmp[ -e trailerlist.tmp ] && rm trailerlist.tmp[ -e trailers.html ] && rm trailers.htmlrm wget-log*# move back to original directorycd "$PWD"