rendered paste body#!/bin/bash# pastes.lst - list of crawled results (sorted and uniqed) (<PastebinID> <Title>)# data/* - paste filesif [ ! -e "data/" ]; then mkdir datafiwhile [ 1 ]; do curl -s "http://pastebin.com/archive" | grep "<td class=\"icon\"><a href=\"/" | sed -E 's/<td class="icon"><a href="\///;s/">/ /;s/<\/a><\/td>//;s/^.{4}//' >> pastes.lst cat pastes.lst > pastes.tmp cat pastes.tmp | sort | uniq > pastes.lst rm pastes.tmp echo "Got new list..." I=0 J=0 LEN="`wc -l pastes.lst | awk '{print $1}'`" while read LINE; do ID="`echo "$LINE" | awk '{print $1}'`" if [ ! -e data/$ID ]; then curl -s -A "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "http://pastebin.com/$ID" | grep "<li class=\"li.\"><div class=\"de.\">" | sed -E 's/^.+class="de.">//;s/<\/div>.+//' | sed 's/ /\t/g' > data/$ID #if [ -n `grep 'Hey, it seems you are requesting a little bit too much from Pastebin. Please slow down!' data/$ID` ]; then # rm data/$ID # J=$(($J+1)) #fi I=$(($I+1)) echo -ne "\rStat: $I/$LEN | Del: $J/$I" fi done < pastes.lst echo "\nGot all Pastes..." echo "Got $I new, $J of them where blocked (deleted)!" sleep 300done