#!/bin/sh
#
# grabdata.sh - Version 3.2
#
# 20/02/2008 - Fixed problem with identifying version# of app used for crunching
# 25/02/2008 - make sure an initial data capture is sorted, not just update ones
#
# Script to retrieve successful task data from the Einstein@Home online database for
# one or many hostIDs (either entered singly or read from a file) and to parse it to
# produce a tabulated set of information (frequency, seq#, cpu_time, app_version) as
# a CSV list for each host of interest.
#
# Hosts entered manually may be added automatically to the HostIDs file for future
# reuse. Extracted results data are added to individual host results files to provide
# an ongoing, updating log of host results statistics.
#
# This script relies on an awk script written by Bikeman.
#
echo
echo "Program to grab stats for successful EAH tasks and write them to a results file."
echo "Data is collected from the EAH website for hosts of interest, using the HostID."
echo "The IDs can be entered manually or read from an existing file named hostids."
echo "HostIDs manually entered will be added to the hostids file if not already there."
echo "The lines in this file contain the HostID and a HostName separated by a space."
hidfile=hostids
mindays=+2
echo
echo "Integer days of enforced wait from previous update before a new one is allowed."
echo -n "(Signed ints like +0 +1 +2 - see -mtime flag on 'find' manpage) (default=+2) : "
read ans
echo
if [ "X$ans" != "X" ]
then
mindays=$ans
fi
#
# Find all results files older than $mindays & store in $hids - ignore newer files
#
hids=`find results.* -mtime $mindays | sed 's/^results\.//'`
#
# If there is no file of HostIDs (ie 1st time run) create a new one
#
if [ ! -f $hidfile ]
then
touch $hidfile
fi
#
# See if we are doing an auto update of all out-of-date host or perhaps a manual run
#
echo
echo -n "Using hosts from hostids file? (y or n - use <cr> if entering manually) : "
read ans
echo
if [ "$ans" == "y" ]
then
if [ "`wc -l $hidfile`" == "0" ]
#
# Can't autorun if there are no hosts filed. Suggest manual host entry
#
then
echo "HostIDs file $hidfile does not contain hosts - use manual entry ..."
echo
exit
else
echo "Any outdated results for hosts in $hidfile file will be retrieved."
echo
fi
else
#
# Manual run. Collect HostIDs. Check if HostID is on the list of out-of-date results
# Only accept it if it is. If it's not, check if we have a results file for it.
# Offer to update HostIDs file if we don't already have this host filed.
#
mhids=""
allhids=`cat $hidfile | sed 's/\ \ .*//'`
while true
do
echo -n "Next hostID to use for results ( <cr> only when finished ) : "
read hid
if [ "X$hid" != "X" ]
then
inhids=n
for i in $hids
do
if [ "$hid" == "$i" ]
then
inhids=y
mhids="$mhids$hid "
break
fi
done
if [ "$inhids" == "n" ]
then
if [ -f results.$hid ]
then
inhids=r
echo "HostID $hid was recently updated -- ignoring this HostID ..."
else
mhids="$mhids$hid "
fi
inallhids=n
for j in $allhids
do
if [ "$j" == "$hid" ]
then
inallhids=y
break
fi
done
if [ "$inallhids" == "n" ]
then
echo -n "Hostname for updating HostIDs file $hidfile (<cr> = Dont update) : "
read host
if [ "X$host" != "X" ]
then
echo "$hid $host" >> $hidfile
fi
else
echo "HostID $hid already in $hidfile - no need to update that file ..."
fi
else
echo "HostID $hid already in $hidfile - no need to update that file ..."
fi
else
break
fi
done
echo "Manually entered HostIDs requiring an update are :-"
echo "$mhids"
echo
echo -n "If these are OK hit <cr> to continue or q to quit : "
read tmp
if [ "X$tmp" != "X" ]
then
exit
else
hids=$mhids
fi
fi
#
# Variable $hids now contains just those hosts (either auto-determined or manually
# entered) that require updating - if there are any.
#
if [ "X$hids" == "X" ]
then
echo "There are no out-of-date results files that need updating -- exiting ... "
echo
exit
fi
#
# Grab the data from the website for each valid host
#
for hid in $hids
do
#
# Get the first page of (up to 20) results Test if there is a "Next" page
#
curl -s "http://einstein.phys.uwm.edu/results.php?hostid=$hid" > results.curl
while true
do
tmp=`grep Next results.curl | tail -1 | sed -e 's/^.*offset.//' -e 's/.Next.*//'`
if [ "X$tmp" != "X" ]
then
#
# There is a next page - keep grabbing until no more
#
while true
do
curl -s "http://einstein.phys.uwm.edu/results.php?hostid=$hid&offset=$tmp" > results.ext
tmp1=`grep Next results.ext | tail -1 | sed -e 's/^.*offset.//' -e 's/.Next.*//'`
cat results.ext >> results.curl
tmp=$tmp1
if [ "X$tmp1" == "X" ]
then
break 2
fi
done
else
break
fi
done
#
# All grabbed pages for a host have been concatenated. Pass through Bikeman's awk script
# Massage to create a CSV list of Freq,Seq#,Runtime and a list of task IDs. Use the TIDs
# to grab the page with stderr.out so that the app version can be obtained.
#
awk -f parser.awk results.curl > results.raw
cut -c-9 results.raw > results.tid
tids=`cat results.tid`
cut -c10- results.raw > results.cut
sed -e s/\ /,/g -e /,[0-9][0-9],/s/,/,0/ -e /,[0-9],/s/,/,00/ -e s/...$// results.cut > results.tmp
paste -d, results.tid results.tmp | sed s/\ // > results.csv
#
# Progress indicator. Each dot means another set of raw host results has been parsed
#
echo -n .
for tid in $tids
do
#
# For each TID obtained from the website, check to see if those details are already recorded
# in the results file, Regard a match of seq# and runtime as sufficient to prove a match.
# Do not grab the page with stderr.out if we already have the data recorded
#
runtime=`grep $tid results.csv | sed 's/^.*,//'`
seqno=`grep $tid results.csv | cut -c18- | sed s/,.*//`
tmp=""
if [ -f results.$hid ]
then
tmp=`grep $runtime results.$hid | grep $seqno`
fi
if [ "X$tmp" == "X" ]
then
#
# No match for seq# and runtime for this TID so we need to grab the page.
#
freq=`grep $tid results.csv | sed -e 's/,[0-9][0-9][0-9],.*//' -e 's/^.*,//'`
ver=`curl -s "http://einstein.phys.uwm.edu/result.php?resultid=$tid" | grep einstein_S5R3 | sed -e 's/^.*ein_S5R3_//' -e 's/_[iwp][6io].*//'`
numv=`echo $ver | wc -w`
#
# Check if more than one app version was used to crunch the data - if so record as 'mixed'
#
if [ $numv != 1 ]
then
flag=1
ver1=`echo $ver | sed 's/\ .*//'`
for i in $ver
do
if [ $i != $ver1 ]
then
flag=2
break
fi
done
if [ $flag == 1 ]
then
ver=$ver1
else
ver=mixed
fi
fi
#
# Assemble a line of new data and store it in a temporary results file
#
echo $freq,$seqno,$runtime,$ver >> results.new
fi
done
#
# Progress indicator. Each plus means another host's new results have been assembled.
#
echo -n +
if [ ! -f results.new ]
then
#
# There are actually no new results at this time so create an empty file as a placeholder
#
touch results.new
fi
if [ -f results.$hid ]
then
#
# If there are existing results for this host add in the new ones. For no existing results
# any new ones will become the future existing ones. Clean up any temp files
#
mv results.$hid results.sav
cat results.sav results.new | sort > results.$hid
rm -f results.sav
else
sort results.new > results.$hid
fi
rm -f results.raw results.c* results.t* results.ext results.new
done
echo