rendered paste body#!/bin/bashCONF="$HOME/.extemppull"#PAPERSALL="0"GUARDIAN="0"MSNBC="0"#OPTIONSUPDATE="0"DOWNLOAD="0"OPEN="0"CLEAN="0"help_msg() { echo "Usage: extemppull [paper] [options]" echo "" echo "Papers:" echo "" echo " all - option applied to all papers" echo " guardian - option applied just to guardian" echo " msnbc - option applied just to msnbc" echo "" echo "Options:" echo "" echo " update - check rss feed for new urls" echo " download - downloaded current url list" echo " open - open with firefox" echo " clean - delete url list" echo "" echo "Any argument that is not one of the above will bring up this." echo "Only one paper and one option at a time, others will be ignored." echo "" exit}err_msg() { echo "$1" exit 2}msnbc_parse() { wget -q "$1" -O- | tr '<' '\n' | grep ^link\> | sed 's/^link>//g' | grep msnbc.msn.com/id/......../$ >> "$URLFILE" [ "$?" -gt "0" ] && err_msg "msnbc update failed."}guardian_parse() { wget -q "$1" -O- | grep \<link\> | grep -v guardian.co.uk/? | sed -e 's/^.*<link>//g' -e 's/<\/link>//g' -e 's/?gusrc=rss.*$//g' | grep -v "/flash/" | grep -v "/video/" | grep -v "/travel/" | grep -v "/fashion/" | grep -v "/championsleague" | grep -v "/motorsport/" | grep .html$ >> $URLFILE [ "$?" -gt "0" ] && err_msg "guardian update failed."} case "$1" in all) ALL="1" ;; guardian) GUARDIAN="1" ;; msnbc) MSNBC="1" ;; *) help_msg ;;esaccase "$2" in update) UPDATE="1" ;; download) DOWNLOAD="1" ;; open) OPEN="1" ;; clean) CLEAN="1" ;; *) help_msg ;;esac#Check for CONF directory[ ! -d "$CONF" ] && mkdir -p "$CONF"#MSNBCif [ "$MSNBC" -eq "1" -o "$ALL" -eq "1" ]; then MSNBCDIR="$HOME/files/school/forensics/extemp/msnbc" URLFILE="$CONF/msnbc" TMPFILE="$CONF/msnbc.tmp" DLEXT="/print/1/displaymode/1098" MSNBCURLS=("http://rss.msnbc.msn.com/id/3032552/device/rss/rss.xml" "http://rss.msnbc.msn.com/id/3032506/device/rss/rss.xml") if [ "$UPDATE" -eq "1" ]; then for url in ${MSNBCURLS[@]}; do msnbc_parse "$url" sort -u "$URLFILE" > "$TMPFILE" rm -f "$URLFILE" mv "$TMPFILE" "$URLFILE" done echo "Updated msnbc." elif [ "$DOWNLOAD" -eq "1" ]; then [ ! -f "$URLFILE" ] && err_msg "No msnbc urls found, try update." [ ! -d "$MSNBCDIR" ] && mkdir -p $MSNBCDIR TOTAL="$(wc -l $URLFILE | awk '{print $1}')" NUM="1" echo "Downloading msnbc." for url in $(cat $URLFILE); do OUTFILE="$(echo $url | grep -Eo [0-9]\{8\}).html" echo -n "Downloading $NUM of $TOTAL..." wget -q ${url}$DLEXT -O $MSNBCDIR/$OUTFILE echo "done." let "NUM++" done elif [ "$OPEN" -eq "1" ]; then [ -z "$(ls $MSNBCDIR)" ] && err_msg "No files in msnbc directory" firefox $MSNBCDIR/* & sleep 3 elif [ "$CLEAN" -eq "1" ]; then [ -f "$URLFILE" ] && rm -f "$URLFILE" [ -d "$MSNBCDIR" ] && [ -n "$(ls $MSNBCDIR)" ] && rm -f $MSNBCDIR/* echo "Cleaned msnbc." fi fi#GUARDIANif [ "$GUARDIAN" -eq "1" -o "$ALL" -eq "1" ]; then GUARDIANDIR="$HOME/files/school/forensics/extemp/guardian" URLFILE="$CONF/guardian" TMPFILE="$CONF/guardian.tmp" GUARDIANURLS=("http://www.guardian.co.uk/rss") BASEURL="http://www.guardian.co.uk" if [ "$UPDATE" -eq "1" ]; then for url in ${GUARDIANURLS[@]}; do guardian_parse "$url" sort -u "$URLFILE" > "$TMPFILE" rm -f "$URLFILE" mv "$TMPFILE" "$URLFILE" done echo "Updated guardian." elif [ "$DOWNLOAD" -eq "1" ]; then [ ! -f "$URLFILE" ] && err_msg "No guardian urls found, try update." [ ! -d "$GUARDIANDIR" ] && mkdir -p $GUARDIANDIR TOTAL="$(wc -l $URLFILE | awk '{print $1}')" NUM="1" echo "Downloading guardian." for url in $(cat $URLFILE); do OUTFILE="$(echo $url | sed 's/.*\///g')" echo -n "Downloading $NUM of $TOTAL..." DLEXT="$(wget -q $url -O- | grep "/print/" | awk -F"\"" '{print $2}')" wget -q ${BASEURL}${DLEXT} -O $GUARDIANDIR/$OUTFILE echo "done." let "NUM++" done elif [ "$OPEN" -eq "1" ]; then [ -z "$(ls $GUARDIANDIR)" ] && err_msg "No files in guardian directory" firefox $GUARDIANDIR/* & elif [ "$CLEAN" -eq "1" ]; then [ -f $URLFILE ] && rm -f $URLFILE [ -d $GUARDIANDIR ] && [ -n "$(ls $GUARDIANDIR)" ] && rm -f $GUARDIANDIR/* echo "Cleaned guardian." fifi