#!/usr/bin/env python# Copyright, 2007, Brian Mingus, Trevor Pincock, Laura Rassbach## This file is part of get_ratings## This library is free software; you can redistribute it and/or# modify it under the terms of the GNU Lesser General Public# License as published by the Free Software Foundation; either# version 2.1 of the License, or (at your option) any later version.## This library is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU# Lesser General Public License for more details. """ Download and create a list of quality articles and the ranked dimensions"""debug = True# Standard Library Imports# For downloading, searching, file operations, stack accessimport datetimeimport urllib2import threadimport sysimport re# Third Party Imports# For walking the DOMfrom BeautifulSoup import BeautifulSoup# So we don't parse all of every pagefrom BeautifulSoup import SoupStrainerclass GetRatings: def debug(self, msg, name, error=False): """ error is a boolean value - is this an error or not? Pass str(sys._getframe().f_code.co_name) as name """ try: if error: print "Error: " + str(name) + ": " + str(msg) elif debug: print "Debug: " + str(name) + ": " + str(msg) except: print "Even your debug statements are buggy!" def URLOpener(self): """ Returns a urllib2 object """ self.opener = urllib2.build_opener() self.opener.addheaders = [('User-agent', 'Mozilla/5.0')] return self.opener def GetHTML(self, url): """ Returns the HTML of a url passed to it """ website = self.URLOpener() website = website.open(url.decode("utf-8")) website = website.read() return website def Beautify(self, html): """ Returns a BeautifulSoup object based on input html """ return BeautifulSoup(html) def GetIndex(self): """ Returns a BeautifulSoup object of the main article quality index at http://en.wikipedia.org/wiki/Wikipedia:Version_1.0_Editorial_Team/Index """ index = self.Beautify(self.GetHTML('http://en.wikipedia.org/wiki/Wikipedia:Version_1.0_Editorial_Team/Index')) return index def isQualityTable(self, page): """ Takes a BeautifulSoup object and determines if it is an article quality table containing the quality metrics of articles It's heuristic is simply the *lack* of a digit in the last character of the URL combined with the existence of the word 'Comments'. """ if not page.html.find(href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+')): if page.html.find(text='Comments'): return True else: return False def isListofQualityTables(self, page): """ Takes a BeautifulSoup object and determines if it is a list of quality metrics of articles (a list of lists) It's heuristic is whether there is a URL whose last character is a digit anywhere on the page """ if page.html.find(href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+')): return True else: return False def GetTitle(self, page): """ Takes a BeautifulSoup object and returns the title of the article """ name = str(sys._getframe().f_code.co_name) # Debugging title_match = '^.*Team\/(.*) - Wikipedia, the free encyclopedia$' title_match = re.compile(title_match) try: title = title_match.search(page.html.title.string).group(1) return title except ValueError: self.debug("Couldnt get the article title from the <title>", name, True) def ParseDumpArticleQualityTable(self, page): """ This function has the unsightly job of parsing the table row by row and writing the output to a CSV We don't care about most of the data it sees - just uncomment whatever else you want """ name = str(sys._getframe().f_code.co_name) # Debugging Article, Importance, Date, Assessment, Version, Comments = '','','','','','' # TODO: str(page) is a major inefficiency in this script table = BeautifulSoup(str(page), parseOnlyThese=SoupStrainer('table', border='1')) trs = table.tr.nextSibling.nextSibling.findNextSiblings('tr') del trs[len(trs) - 1] # Last item is extraneous date = datetime.date.today() f = open('ETQR' + str(date) + '.csv', 'a') write = True for i in range(len(trs)): pointer = trs[i].findNext('td') try: Article = pointer.next.string # Article except: write = False pointer = pointer.findNext('td') #Importance = pointer.next.string pointer = pointer.findNext('td') # Date = pointer.next.string pointer = pointer.findNext('td') # Assessment try: Assessment = pointer.next.a.string # Odd man out except: # This is for FA/GA articles try: Assessment = pointer.findNext('b').findNext('a').string except: write = False pointer = pointer.findNext('td') # Version = pointer.next.string pointer = pointer.findNext('td') # Comments = pointer.next.string if write: # Only log thesex if Assessment in ['FA','A', 'GA', 'B', 'Start']: data = str(Article) + '____' + str(Assessment) data = data.encode("utf-8") + '\n' self.debug("Write: " + str(write) + ": " + data, name) f.write(data) f.close() def ParseDumpListOfQualityTables(self, page): """ Knows how to parse a LIST of quality tables, and then call ParseDumpArticleQualityTable on them """ name = str(sys._getframe().f_code.co_name) # Debugging links = page.html.findAll('a', href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+')) title = self.GetTitle(page) self.debug("len(links): " + str(len(links)), name) for i in range(len(links)): try: url = 'http://en.wikipedia.org' + str(links[i]['href']).decode("utf-8") except: return 0 try: new_page = self.Beautify(self.GetHTML(url).decode("utf-8")) except: self.debug("Check page ", name, True) try: thread.start_new_thread(self.ParseDumpArticleQualityTable, (new_page,)) #self.ParseDumpArticleQualityTable(new_page) #self.WriteToCSVRecursive(new_page) except: self.debug("Exception in " + str(title), name, True) def ParseIndex(self, index): """ Parses the main index and returns a BeautifulSoup list of all the appropriate <a href=""></a> tags """ # This gets n - 1 articles - small bug. index_list = index.html.findAllNext('a',href=re.compile('.*articles_by_quality.*'),text=re.compile('.*articles by quality.*')) return index_list def WriteToCSVRecursive(self, page): """ Not actually recursive, but if it's actually needed in the future it almost works. Uncomment below... """ name = str(sys._getframe().f_code.co_name) # Debugging title = self.GetTitle(page) self.debug('', name) if self.isQualityTable(page): self.debug("Table: " + title, name) thread.start_new_thread(self.ParseDumpArticleQualityTable, (page,)) #self.ParseDumpArticleQualityTable(page) # ParseDumpListOfQualityTables calls WriteToCSVRecursive elif self.isListofQualityTables(page): self.debug("List: " + title, name) self.ParseDumpListOfQualityTables(page) else: self.debug("Unknown: " + title, name) def GetList(self): """ 1. Download the index 2. Parse it with BeautifulSoup to get a list of articles 3. Determine if we have a list of articles or a list of lists of articles """ index = self.GetIndex() # 1. list_of_article_titles = self.ParseIndex(index) # 2. for i in range(len(list_of_article_titles)): # 3. url = 'http://en.wikipedia.org' + list_of_article_titles[i].parent['href'] page = self.Beautify(self.GetHTML(url.decode("utf-8"))) self.WriteToCSVRecursive(page) def main(self): self.GetList()if __name__ == "__main__": articleList = GetRatings() sys.exit(articleList.main())