#!/bin/bash#Author: bruenig#This script provides a unified way to interface with newspapers for extempDB="$HOME/.extemp"CONFFILE='/etc/extemp/extemp.conf'LIB='/usr/lib/extemp'OPTIONS=('load' 'clean' 'search' 'status' 'update')SPECIALPAPERS=('all')COREUTILDEPS=('cp' 'du' 'mkdir' 'printf' 'rm' 'sleep' 'stty' 'test' 'tr' 'uniq' 'wc')OTHERDEPS=('awk' 'find' 'firefox' 'grep' 'sed' 'wget') #Message functionserr_msg() { echo "$1" exit 1}help_msg() { width="$(stty size | awk '{print $2}')" papernum="$(wc -w <<<${PAPERS[@]})" paperlen='0' for thepapers in "${PAPERS[@]}"; do if [ "${#thepapers}" -gt "$paperlen" ]; then paperlen="${#thepapers}" fi done wordsperline="$((width/(paperlen+1)))" echo "Usage: extemp [option] [paper] \"[search phrase]\"" echo echo "Options:" echo echo " clean - delete old downloaded articles" echo " load - load database into cache for faster searching" echo " search - search database for \`search phrase'" echo " status - print the status of the database" echo " update - download new articles into the database" echo echo "Papers:" echo echo "all - applies option all papers below" echo if [ "$wordsperline" -eq "0" ]; then echo "${PAPERS[@]}" else wordnum="0" for paper in "${PAPERS[@]}"; do let wordnum++ if [ "$wordnum" -eq "$wordsperline" ]; then echo "$paper" wordnum="0" else echo -n "$paper " thislen="${#paper}" spaces="$((paperlen+1-thislen))" for ((i=0; i<$spaces; i++)); do echo -n " " done fi done fi echo exit}#Sanity checks#Dependency checksfor dependency in "${COREUTILDEPS[@]}"; do if ! which $dependency &> /dev/null; then err_msg "Program \`$dependency' of coreutils needed to run." fidonefor dependency in "${OTHERDEPS[@]}"; do if ! which $dependency &> /dev/null; then err_msg "Program \`$dependency' is needed to run." fidone#No root allowed[ "$UID" -eq "0" ] && err_msg "This program cannot be executed as root."#Check for DB directory[ ! -d "$DB" ] && mkdir -p "$DB"#Check for CONFFILE, make it if it does not exist, source it[ ! -f "$CONFFILE" ] && err_msg "The file \`$CONFFILE' does not exist". "$CONFFILE"#Fill PAPERS array and source paper function filesNUM="0"for file in ${LIB}/*; do . ${file} PAPERS[$NUM]=${file##*/} let NUM++done#Parameter handlingcase "$1" in ''|-h|--help|help) help_msg;; *) ;;esacfor availoption in "${OPTIONS[@]}"; do if [ "$1" = "$availoption" ]; then option="$1" CORRECTOPTION="1" break fidone[ "$CORRECTOPTION" = "1" ] || err_msg "Bad option \`$1', see extemp help."for availpaper in "${PAPERS[@]}" "${SPECIALPAPERS[@]}"; do if [ "$2" = "$availpaper" ]; then paper="$2" CORRECTPAPER="1" break fidone[ "$CORRECTPAPER" = "1" ] || err_msg "Bad paper \`$2', see extemp help."#Common Functionsupdate() { PAPER="$1" [ ! -d "$DLDIR" ] && mkdir -p $DLDIR [ -f "$URLFILE" ] && rm -f "$URLFILE" echo echo "##### $PAPER #####" ${PAPER}_update if [ "$?" -eq "0" ]; then echo "Updated ${PAPER}." else echo "Failed to update ${PAPER}." fi rm -f "$URLFILE"}clean() { [ -z "$CLEANDAYS" ] && err_msg "CLEANDAYS not set in extemp.conf" PAPER="$1" deletedfiles='0' cd $DLDIR for file in $(find . -mtime +"$CLEANDAYS" -type f); do rm -f $file let deletedfiles++ done deletedlen="${#deletedfiles}" spacesnum=$((5-deletedlen)) echo -n "Deleted $deletedfiles" for ((x=0; x<$spacesnum; x++)); do echo -n " " done echo "old articles from $PAPER."}status() { PAPER="$1" if [ "$paper" = "all" ]; then paperlen='0' for thepapers in "${PAPERS[@]}"; do if [ "${#thepapers}" -gt "$paperlen" ]; then paperlen="${#thepapers}" fi done for thepapers in "${PAPERS[@]}" "TOTAL"; do thislen="${#thepapers}" firstspacesnum=$((paperlen+1-thislen)) if [ "$thepapers" = "TOTAL" ]; then cd "$DB/articles" echo #Create space between papers and TOTAL else cd "$DB/articles/$thepapers" fi filesnum="$(find . -type f | wc -l)" disknum="$(du -ha | awk '/\.$/ {print $1}')" templenvar="${#filesnum}" secondspacesnum=$((6-templenvar)) echo -n "$thepapers:" for ((i=0; i<$firstspacesnum; i++)); do echo -n " " done echo -n "$filesnum" for ((i=0; i<$secondspacesnum; i++)); do echo -n " " done echo "articles $disknum" done exit else cd "$DLDIR" filesnum="$(find . -type f | wc -l)" disknum="$(du -ha | awk '/\.$/ {print $1}')" echo "$PAPER: $filesnum articles $disknum" fi}#Execution#search handlingif [ "$option" = "search" ]; then shift 2 [ -z "$1" ] && err_msg "No search phrases were specified, see extemp help." if [ "$paper" = "all" ]; then SEARCHDIR="$DB/articles" else SEARCHDIR="$DB/articles/$paper" fi [ -d "$SEARCHDIR" ] || err_msg "No database entry for \`$paper'" echo "Searching database..." firstpassdone='0' for string in "$@"; do if [ "$firstpassdone" = "0" ]; then articles="$(grep -iRl "$string" $SEARCHDIR/*)" firstpassdone='1' else articles="$(grep -il "$string" $articles)" fi [ -z "$articles" ] && err_msg "Matching failed at \`$string'" done articlematches="$(echo "$articles" | wc -l)" if [ "$articlematches" -ge "$MATCHLIMIT" ]; then echo -n "$articlematches matches, open all of them? [y/n] " read -en 1 answer case "$answer" in 'y'|'Y') ;; *) echo "Narrow the matches with more/different search terms this time." exit ;; esac fi firefox $articles & if [ "$articlematches" -lt 50 ]; then echo "$articlematches matches." else echo "Be patient: $articlematches articles will take some time to load." fi exitfi#load handling#this is really just a dirty hack#it greps through all the articles so they get cached first using searchif [ "$option" = "load" ]; then echo "Loading can take a couple of minutes so be patient:" echo -n "Loading $paper..." extemp search "$paper" "asdfasfdasdfasf" <<<n &> /dev/null echo "done" exitfi#status, clean, update handlingif [ "$paper" = "all" ]; then for temppaper in "${PAPERS[@]}"; do DLDIR="$DB/articles/$temppaper" [ -d "$DLDIR" ] || mkdir -p "$DLDIR" URLFILE="$DLDIR/$temppaper.url" $option $temppaper doneelse #just one explicit paper DLDIR="$DB/articles/$paper" [ -d "$DLDIR" ] || mkdir -p "$DLDIR" URLFILE="$DLDIR/$paper.url" $option $paperfi