#! /usr/bin/python
# -*- coding: utf-8 -*-
import sys, os, re, urllib, csv
re_successes = 0
re_fails = 0
re_none = 0
no = 0
verbose = False
def uniqify(list):
uniques = []
for item in list:
if item not in uniques:
uniques.append(item)
return uniques
def article_parse(conf_name, articlenr, article, writer):
global re_successes, re_fails, re_none, no
global verbose
print article, articlenr
sys.exit()
no += 1
if no % 10 == 0:
if not verbose: print '\x08'*80, "Processing #%d." % no,
# print conf_name, article
fullfile = ''.join(open(article).readlines())
# Extract the title, with some sanity checks
m = re.search(r"<title>(.*?)</title>", fullfile)
if not m: return
title = m.group(1)
if verbose: print '\t', title
if verbose: print '\t', article
subconf_name = ''
title_parts = title.split(',', 1)
if not ':' in title or not ',' in title:
title = title_parts[0]
conf_name, subconf_name = conf_name.split('_')
else:
conf, title = title_parts
subconf_name = conf.split(':', 1)[1]
conf_name = conf_name.split('_')[0]
title = title.strip()
if title[-3:] == '...': title = title[:-3]
subconf_name = subconf_name.strip()
conf_name = conf_name.strip()
# Don't want too long strings
if len(title) < 20:
title = title[0:]
else:
title = title[:20]
if verbose: print conf_name, subconf_name, title
# les hele filen inn til en string
try:
fullfile = fullfile[fullfile.index('<p class="abstract"><font color="Red">Note: OCR errors may be found in this Reference List extracted from the full text article. ACM has opted to expose the complete List rather than only correct and linked references.</font></p>'):]
fullfile = fullfile[:fullfile.index('</table>')]
except:
return
references = fullfile.split('<div class="abstract">')[1:]
refnr = []
refyears= []
# Every second <div class="abstract"> is a reference
isRef = False
for ref in references:
isRef = not isRef
if isRef:
nr = ref[:ref.index('</div>')].strip()
refnr.append(nr)
else:
ref_text = ref[:ref.index('</div>')].strip()
ref_text_old = ref_text
# if we have a link in the text, try extracting the contents of the tag
if 'a href' in ref_text:
ref_start = ref_text.find('>')
ref_stop = ref_text.find('</a>', ref_start)
ref_text = ref_text[ref_start+1:ref_stop].strip()
# writer.write(ref_text + '\n')
# for x in ["Published", "accessed"]:
# if x in ref_text:
# ref_text = ref_text[:ref_text.find(x)]
# break
else:
# if we have a failpedia or failttp reference, skip the ref
if "http" in ref_text: continue
if "ikipedia" in ref_text: continue
# regex: (start of line or not (digit or letter) followed by 19 or 20 followed by 2 digits followed by end of line or not a digit.
pattern = r"([^0-9a-zA-Z]|^)((19|20)\d\d)([^0-9]|$)"
all_matches = re.findall(pattern, ref_text )
m = map(lambda x: x[1], all_matches) # scramble array into a usable form
if m:
# Treat manually
if len(uniqify(m)) > 1:
# Get the refnr that failed, and put it back
refn = refnr.pop()
refnr.append(refn)
refyears.append('-')
writer.writerow([title, int(articlenr), int(refn), "CHECK ME", "FILL ME", ref_text])
re_fails += 1
# writer.write(ref_text + '\n')
# writer.write(ref_text+ '\n')
# writer.write(ref_text_old+ '\n')
# writer.write(":".join(m)+'\n')
# writer.write("*"*78 + '\n')
print "Check manually as", m
# Same year in the ref, probably okay
elif len(uniqify(m)) == 1:
refn = refnr.pop()
refnr.append(refn)
age = str(2008-int(m[0][0]))
writer.writerow([title, int(articlenr), int(refn), int(m[0][0]), int(age)])
refyears.append(m[0][0])
print "found one", m[0][0]
re_successes += 1
else:
print "SHOULD NOT HAPPEN!"
sys.exit(0)
else:
print "No reference"
re_none += 1
refn = refnr.pop()
refnr.append(refn)
refyears.append('-')
writer.writerow([title, articlenr, refn," "," "])
return
if __name__ == '__main__':
#filter = ''
#if len(sys.argv) == 2:
# filter = sys.argv[1]
for conference in os.listdir("."):
# if filter and conference != filter: continue
if os.path.isdir(conference):
for sub_conference in os.listdir(conference):
file_path = "/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference+"/"+'_'.join((conference, sub_conference))+"_result.csv"
if not os.path.isdir("/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference):
os.makedirs("/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference)
output = csv.writer(open(file_path, "w"))
for article in os.listdir(os.path.join(conference,sub_conference)):
article_parse('_'.join((conference, sub_conference)), article, os.path.join(conference, sub_conference, article), output)
print '\nsuccesses:\t', re_successes
print 'fails:\t\t', re_fails
print 'none:\t\t', re_none
print '%\t\t', re_fails * 100 / (re_fails + re_successes+re_none)
output.close()